agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. 157+ messages / 2 participants [nested] [flat]
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH 2/2] Add the "dummy_table_am" test module @ 2025-03-01 19:50 Julien Tachoires <julien@tachoires.me> 0 siblings, 0 replies; 157+ messages in thread From: Julien Tachoires @ 2025-03-01 19:50 UTC (permalink / raw) This test module is in charge of testing table AM reloptions. It's very similar to what we do in dummy_index_am as we have to exercise the exact same kind of feature. --- src/test/modules/Makefile | 1 + src/test/modules/dummy_table_am/Makefile | 20 + src/test/modules/dummy_table_am/README | 14 + .../dummy_table_am/dummy_table_am--1.0.sql | 13 + .../modules/dummy_table_am/dummy_table_am.c | 588 ++++++++++++++++++ .../dummy_table_am/dummy_table_am.control | 5 + .../dummy_table_am/expected/reloptions.out | 181 ++++++ src/test/modules/dummy_table_am/meson.build | 33 + .../modules/dummy_table_am/sql/reloptions.sql | 99 +++ src/test/modules/meson.build | 1 + 10 files changed, 955 insertions(+) create mode 100644 src/test/modules/dummy_table_am/Makefile create mode 100644 src/test/modules/dummy_table_am/README create mode 100644 src/test/modules/dummy_table_am/dummy_table_am--1.0.sql create mode 100644 src/test/modules/dummy_table_am/dummy_table_am.c create mode 100644 src/test/modules/dummy_table_am/dummy_table_am.control create mode 100644 src/test/modules/dummy_table_am/expected/reloptions.out create mode 100644 src/test/modules/dummy_table_am/meson.build create mode 100644 src/test/modules/dummy_table_am/sql/reloptions.sql diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 4e4be3fa511..8fe2a2904d6 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -9,6 +9,7 @@ SUBDIRS = \ commit_ts \ delay_execution \ dummy_index_am \ + dummy_table_am \ dummy_seclabel \ libpq_pipeline \ oauth_validator \ diff --git a/src/test/modules/dummy_table_am/Makefile b/src/test/modules/dummy_table_am/Makefile new file mode 100644 index 00000000000..94837dff392 --- /dev/null +++ b/src/test/modules/dummy_table_am/Makefile @@ -0,0 +1,20 @@ +# src/test/modules/dummy_table_am/Makefile + +MODULES = dummy_table_am + +EXTENSION = dummy_table_am +DATA = dummy_table_am--1.0.sql +PGFILEDESC = "dummy_table_am - table access method template" + +REGRESS = reloptions + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/dummy_table_am +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/dummy_table_am/README b/src/test/modules/dummy_table_am/README new file mode 100644 index 00000000000..50cf08ee3b1 --- /dev/null +++ b/src/test/modules/dummy_table_am/README @@ -0,0 +1,14 @@ +Dummy Table AM +============== + +Dummy table AM is a module for testing any facility usable by a table +access method, whose code is kept a maximum simple. + +This includes tests for all relation option types: +- boolean +- enum +- integer +- real +- strings (with and without NULL as default) + +It also includes tests related to unrecognized options. diff --git a/src/test/modules/dummy_table_am/dummy_table_am--1.0.sql b/src/test/modules/dummy_table_am/dummy_table_am--1.0.sql new file mode 100644 index 00000000000..12ad3ad174b --- /dev/null +++ b/src/test/modules/dummy_table_am/dummy_table_am--1.0.sql @@ -0,0 +1,13 @@ +/* src/test/modules/dummy_table_am/dummy_table_am--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION dummy_table_am" to load this file. \quit + +CREATE FUNCTION dummy_table_am_handler(internal) +RETURNS table_am_handler +AS 'MODULE_PATHNAME' +LANGUAGE C; + +-- Access method +CREATE ACCESS METHOD dummy_table_am TYPE TABLE HANDLER dummy_table_am_handler; +COMMENT ON ACCESS METHOD dummy_table_am IS 'Dummy Table Access Method'; diff --git a/src/test/modules/dummy_table_am/dummy_table_am.c b/src/test/modules/dummy_table_am/dummy_table_am.c new file mode 100644 index 00000000000..a473bc7dd9b --- /dev/null +++ b/src/test/modules/dummy_table_am/dummy_table_am.c @@ -0,0 +1,588 @@ +/*------------------------------------------------------------------------- + * + * dummy_table_am.c + * Table AM templae main file + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/test/modules/dummy_table_am/dummy_table_am.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "miscadmin.h" + +#include "access/hio.h" +#include "access/relscan.h" +#include "access/reloptions.h" +#include "access/tableam.h" +#include "access/sdir.h" +#include "access/skey.h" +#include "executor/tuptable.h" +#include "utils/relcache.h" +#include "utils/snapshot.h" + + +PG_MODULE_MAGIC; + +/* Base structures for scans */ +typedef struct DummyScanDescData +{ + TableScanDescData rs_base; /* AM independent part of the descriptor */ + + /* Add more fields here as needed by the AM. */ +} DummyScanDescData; +typedef struct DummyScanDescData *DummyScanDesc; + +/* parse table for fillRelOptions */ +static relopt_parse_elt dt_relopt_tab[7]; + +/* Kind of relation options for dummy index */ +static relopt_kind dt_relopt_kind; + +typedef enum DummyAmEnum +{ + DUMMY_AM_ENUM_ONE, + DUMMY_AM_ENUM_TWO, +} DummyAmEnum; + +/* Dummy table options */ +typedef struct DummyTableOptions +{ + int32 vl_len_; /* varlena header (do not touch directly!) */ + int option_int; + double option_real; + bool option_bool; + DummyAmEnum option_enum; + int option_string_val_offset; + int option_string_null_offset; + int fillfactor; +} DummyTableOptions; + +static relopt_enum_elt_def dummyAmEnumValues[] = +{ + {"one", DUMMY_AM_ENUM_ONE}, + {"two", DUMMY_AM_ENUM_TWO}, + {(const char *) NULL} /* list terminator */ +}; + +/* ------------------------------------------------------------------------ + * Dummy Access Method Interface + * ------------------------------------------------------------------------ + */ + +static const TupleTableSlotOps * +dummy_slot_callbacks(Relation relation) +{ + return &TTSOpsMinimalTuple; +} + +static TableScanDesc +dummy_scan_begin(Relation relation, Snapshot snapshot, int nkeys, ScanKey key, + ParallelTableScanDesc parallel_scan, uint32 flags) +{ + DummyScanDesc scan; + + scan = (DummyScanDesc) palloc(sizeof(DummyScanDescData)); + + scan->rs_base.rs_rd = relation; + scan->rs_base.rs_snapshot = snapshot; + scan->rs_base.rs_nkeys = nkeys; + scan->rs_base.rs_flags = flags; + scan->rs_base.rs_parallel = parallel_scan; + + return (TableScanDesc) scan; +} + +static void +dummy_scan_end(TableScanDesc sscan) +{ + DummyScanDesc scan = (DummyScanDesc) sscan; + + pfree(scan); + + return; +} + +static void +dummy_scan_rescan(TableScanDesc sscan, ScanKey key, bool set_params, + bool allow_strat, bool allow_sync, bool allow_pagemode) +{ + return; +} + +static bool +dummy_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, + TupleTableSlot *slot) +{ + return true; +} + +static void +dummy_scan_set_tidrange(TableScanDesc sscan, ItemPointer mintid, + ItemPointer maxtid) +{ + return; +} + +static bool +dummy_scan_getnextslot_tidrange(TableScanDesc sscan, ScanDirection direction, + TupleTableSlot *slot) +{ + return true; +} + +static Size +dummy_parallelscan_estimate(Relation rel) +{ + return 0; +} + +static Size +dummy_parallelscan_initialize(Relation rel, ParallelTableScanDesc pscan) +{ + return 0; +} + +static void +dummy_parallelscan_reinitialize(Relation rel, ParallelTableScanDesc pscan) +{ + return; +} + +static IndexFetchTableData * +dummy_index_fetch_begin(Relation rel) +{ + return NULL; +} + +static void +dummy_index_fetch_reset(IndexFetchTableData *scan) +{ + return; +} + +static void +dummy_index_fetch_end(IndexFetchTableData *scan) +{ + return; +} + +static bool +dummy_index_fetch_tuple(struct IndexFetchTableData *scan, ItemPointer tid, + Snapshot snapshot, TupleTableSlot *slot, + bool *call_again, bool *all_dead) +{ + return true; +} + +static void +dummy_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, + int options, BulkInsertStateData *bistate) +{ + DummyTableOptions *relopts; + + relopts = (DummyTableOptions *) relation->rd_options; + + elog(NOTICE, "option_int=%d, option_real=%f, option_bool=%d, option_enum=%d", + relopts->option_int, relopts->option_real, relopts->option_bool, relopts->option_enum); + + return; +} + +static void +dummy_tuple_insert_speculative(Relation relation, TupleTableSlot *slot, + CommandId cid, int options, + BulkInsertStateData *bistate, uint32 specToken) +{ + return; +} + +static void +dummy_tuple_complete_speculative(Relation relation, TupleTableSlot *slot, + uint32 specToken, bool succeeded) +{ + return; +} + +static void +dummy_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, + CommandId cid, int options, BulkInsertStateData *bistate) +{ + return; +} + +static TM_Result +dummy_tuple_delete(Relation relation, ItemPointer tid, CommandId cid, + Snapshot snapshot, Snapshot crosscheck, bool wait, + TM_FailureData *tmfd, bool changingPart) +{ + return TM_Ok; +} + +static TM_Result +dummy_tuple_update(Relation relation, ItemPointer otid, TupleTableSlot *slot, + CommandId cid, Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd, + LockTupleMode *lockmode, TU_UpdateIndexes *update_indexes) +{ + return TM_Ok; +} + +static TM_Result +dummy_tuple_lock(Relation relation, ItemPointer tid, Snapshot snapshot, + TupleTableSlot *slot, CommandId cid, LockTupleMode mode, + LockWaitPolicy wait_policy, uint8 flags, + TM_FailureData *tmfd) +{ + return TM_Ok; +} + +static bool +dummy_fetch_row_version(Relation relation, ItemPointer tid, + Snapshot snapshot, TupleTableSlot *slot) +{ + return false; +} + +static void +dummy_get_latest_tid(TableScanDesc sscan, ItemPointer tid) +{ + return; +} + +static bool +dummy_tuple_tid_valid(TableScanDesc scan, ItemPointer tid) +{ + return false; +} + +static bool +dummy_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot, + Snapshot snapshot) +{ + return false; +} + +static TransactionId +dummy_index_delete_tuples(Relation rel, TM_IndexDeleteOp *delstate) +{ + return InvalidTransactionId; +} + +static void +dummy_relation_set_new_filelocator(Relation rel, + const RelFileLocator *newrlocator, + char persistence, + TransactionId *freezeXid, + MultiXactId *minmulti) +{ + return; +} + +static void +dummy_relation_nontransactional_truncate(Relation rel) +{ + return; +} + +static void +dummy_relation_copy_data(Relation rel, const RelFileLocator *newrlocator) +{ + return; +} + +static void +dummy_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, + Relation OldIndex, bool use_sort, + TransactionId OldestXmin, + TransactionId *xid_cutoff, + MultiXactId *multi_cutoff, + double *num_tuples, + double *tups_vacuumed, + double *tups_recently_dead) +{ + return; +} + +static void +dummy_relation_vacuum(Relation rel, struct VacuumParams *params, + BufferAccessStrategy bstrategy) +{ + return; +} + +static bool +dummy_scan_analyze_next_block(TableScanDesc scan, ReadStream *stream) +{ + return false; +} + +static bool +dummy_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, + double *liverows, double *deadrows, + TupleTableSlot *slot) +{ + return false; +} + +static double +dummy_index_build_range_scan(Relation heapRelation, + Relation indexRelation, + struct IndexInfo *indexInfo, + bool allow_sync, + bool anyvisible, + bool progress, + BlockNumber start_blockno, + BlockNumber numblocks, + IndexBuildCallback callback, + void *callback_state, + TableScanDesc scan) +{ + return 0; +} + +static void +dummy_index_validate_scan(Relation heapRelation, + Relation indexRelation, + struct IndexInfo *indexInfo, + Snapshot snapshot, + struct ValidateIndexState *state) +{ + return; +} + +static uint64 +dummy_relation_size(Relation rel, ForkNumber forkNumber) +{ + return 0; +} + +static bool +dummy_relation_needs_toast_table(Relation rel) +{ + return false; +} + +static Oid +dummy_relation_toast_am(Relation rel) +{ + return InvalidOid; +} + +static void +dummy_relation_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, + int32 sliceoffset, int32 slicelength, + struct varlena *result) +{ + return; +} + +static void +dummy_relation_estimate_size(Relation rel, int32 *attr_widths, + BlockNumber *pages, double *tuples, + double *allvisfrac) +{ + return; +} + +static bool +dummy_scan_bitmap_next_block(TableScanDesc scan, BlockNumber *blockno, + bool *recheck, uint64 *lossy_pages, + uint64 *exact_pages) +{ + return false; +} + +static bool +dummy_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) +{ + return false; +} + +static bool +dummy_scan_sample_next_block(TableScanDesc scan, struct SampleScanState *scanstate) +{ + return false; +} + +static bool +dummy_scan_sample_next_tuple(TableScanDesc scan, struct SampleScanState *scanstate, + TupleTableSlot *slot) +{ + return false; +} + +static bytea * +dummy_relation_options(char relkind, Datum reloptions, bool validate) +{ + return (bytea *) build_reloptions(reloptions, validate, + dt_relopt_kind, + sizeof(DummyTableOptions), + dt_relopt_tab, lengthof(dt_relopt_tab)); +} + +/* + * Validation function for string relation options. + */ +static void +validate_string_option(const char *value) +{ + ereport(NOTICE, + (errmsg("new option value for string parameter %s", + value ? value : "NULL"))); +} + +/* + * This function creates a full set of relation option types, + * with various patterns. + */ +static void +create_reloptions_table(void) +{ + dt_relopt_kind = add_reloption_kind(); + + add_int_reloption(dt_relopt_kind, "option_int", + "Integer option for dummy_table_am", + 10, -10, 100, AccessExclusiveLock); + dt_relopt_tab[0].optname = "option_int"; + dt_relopt_tab[0].opttype = RELOPT_TYPE_INT; + dt_relopt_tab[0].offset = offsetof(DummyTableOptions, option_int); + + add_real_reloption(dt_relopt_kind, "option_real", + "Real option for dummy_table_am", + 3.1415, -10, 100, AccessExclusiveLock); + dt_relopt_tab[1].optname = "option_real"; + dt_relopt_tab[1].opttype = RELOPT_TYPE_REAL; + dt_relopt_tab[1].offset = offsetof(DummyTableOptions, option_real); + + add_bool_reloption(dt_relopt_kind, "option_bool", + "Boolean option for dummy_table_am", + true, AccessExclusiveLock); + dt_relopt_tab[2].optname = "option_bool"; + dt_relopt_tab[2].opttype = RELOPT_TYPE_BOOL; + dt_relopt_tab[2].offset = offsetof(DummyTableOptions, option_bool); + + add_enum_reloption(dt_relopt_kind, "option_enum", + "Enum option for dummy_table_am", + dummyAmEnumValues, + DUMMY_AM_ENUM_ONE, + "Valid values are \"one\" and \"two\".", + AccessExclusiveLock); + dt_relopt_tab[3].optname = "option_enum"; + dt_relopt_tab[3].opttype = RELOPT_TYPE_ENUM; + dt_relopt_tab[3].offset = offsetof(DummyTableOptions, option_enum); + + add_string_reloption(dt_relopt_kind, "option_string_val", + "String option for dummy_table_am with non-NULL default", + "DefaultValue", &validate_string_option, + AccessExclusiveLock); + dt_relopt_tab[4].optname = "option_string_val"; + dt_relopt_tab[4].opttype = RELOPT_TYPE_STRING; + dt_relopt_tab[4].offset = offsetof(DummyTableOptions, + option_string_val_offset); + + /* + * String option for dummy_table_am with NULL default, and without + * description. + */ + add_string_reloption(dt_relopt_kind, "option_string_null", + NULL, /* description */ + NULL, &validate_string_option, + AccessExclusiveLock); + dt_relopt_tab[5].optname = "option_string_null"; + dt_relopt_tab[5].opttype = RELOPT_TYPE_STRING; + dt_relopt_tab[5].offset = offsetof(DummyTableOptions, + option_string_null_offset); + + /* + * fillfactor will be used to check reloption conversion when changing + * table access method between heap AM and dummy_table_am. + */ + add_int_reloption(dt_relopt_kind, "fillfactor", + "Fillfactor option for dummy_table_am", + 10, 0, 90, AccessExclusiveLock); + dt_relopt_tab[6].optname = "fillfactor"; + dt_relopt_tab[6].opttype = RELOPT_TYPE_INT; + dt_relopt_tab[6].offset = offsetof(DummyTableOptions, fillfactor); +} + + +/* + * Table Access Method API + */ +static const TableAmRoutine dummy_table_am_methods = { + .type = T_TableAmRoutine, + + .slot_callbacks = dummy_slot_callbacks, + .scan_begin = dummy_scan_begin, + .scan_end = dummy_scan_end, + .scan_rescan = dummy_scan_rescan, + .scan_getnextslot = dummy_scan_getnextslot, + + .scan_set_tidrange = dummy_scan_set_tidrange, + .scan_getnextslot_tidrange = dummy_scan_getnextslot_tidrange, + + .parallelscan_estimate = dummy_parallelscan_estimate, + .parallelscan_initialize = dummy_parallelscan_initialize, + .parallelscan_reinitialize = dummy_parallelscan_reinitialize, + + .index_fetch_begin = dummy_index_fetch_begin, + .index_fetch_reset = dummy_index_fetch_reset, + .index_fetch_end = dummy_index_fetch_end, + .index_fetch_tuple = dummy_index_fetch_tuple, + + .tuple_insert = dummy_tuple_insert, + .tuple_insert_speculative = dummy_tuple_insert_speculative, + .tuple_complete_speculative = dummy_tuple_complete_speculative, + .multi_insert = dummy_multi_insert, + .tuple_delete = dummy_tuple_delete, + .tuple_update = dummy_tuple_update, + .tuple_lock = dummy_tuple_lock, + + .tuple_fetch_row_version = dummy_fetch_row_version, + .tuple_get_latest_tid = dummy_get_latest_tid, + .tuple_tid_valid = dummy_tuple_tid_valid, + .tuple_satisfies_snapshot = dummy_tuple_satisfies_snapshot, + .index_delete_tuples = dummy_index_delete_tuples, + + .relation_set_new_filelocator = dummy_relation_set_new_filelocator, + .relation_nontransactional_truncate = dummy_relation_nontransactional_truncate, + .relation_copy_data = dummy_relation_copy_data, + .relation_copy_for_cluster = dummy_relation_copy_for_cluster, + .relation_vacuum = dummy_relation_vacuum, + .scan_analyze_next_block = dummy_scan_analyze_next_block, + .scan_analyze_next_tuple = dummy_scan_analyze_next_tuple, + .index_build_range_scan = dummy_index_build_range_scan, + .index_validate_scan = dummy_index_validate_scan, + + .relation_size = dummy_relation_size, + .relation_needs_toast_table = dummy_relation_needs_toast_table, + .relation_toast_am = dummy_relation_toast_am, + .relation_fetch_toast_slice = dummy_relation_fetch_toast_slice, + .relation_estimate_size = dummy_relation_estimate_size, + .relation_options = dummy_relation_options, + + .scan_bitmap_next_block = dummy_scan_bitmap_next_block, + .scan_bitmap_next_tuple = dummy_scan_bitmap_next_tuple, + .scan_sample_next_block = dummy_scan_sample_next_block, + .scan_sample_next_tuple = dummy_scan_sample_next_tuple +}; + +PG_FUNCTION_INFO_V1(dummy_table_am_handler); + +Datum +dummy_table_am_handler(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(&dummy_table_am_methods); +} + +void +_PG_init(void) +{ + create_reloptions_table(); +} diff --git a/src/test/modules/dummy_table_am/dummy_table_am.control b/src/test/modules/dummy_table_am/dummy_table_am.control new file mode 100644 index 00000000000..08f2f868d49 --- /dev/null +++ b/src/test/modules/dummy_table_am/dummy_table_am.control @@ -0,0 +1,5 @@ +# dummy_table_am extension +comment = 'dummy_table_am - table access method template' +default_version = '1.0' +module_pathname = '$libdir/dummy_table_am' +relocatable = true diff --git a/src/test/modules/dummy_table_am/expected/reloptions.out b/src/test/modules/dummy_table_am/expected/reloptions.out new file mode 100644 index 00000000000..0b947500ead --- /dev/null +++ b/src/test/modules/dummy_table_am/expected/reloptions.out @@ -0,0 +1,181 @@ +-- Tests for relation options +CREATE EXTENSION dummy_table_am; +CREATE TABLE dummy_test_tab (i int4) USING dummy_table_am; +-- Silence validation checks for strings +SET client_min_messages TO 'warning'; +-- Test with default values. +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +-------- +(0 rows) + +DROP TABLE dummy_test_tab; +-- Test with full set of options. +-- Allow validation checks for strings +SET client_min_messages TO 'notice'; +CREATE TABLE dummy_test_tab (i int4) + USING dummy_table_am WITH ( + option_bool = false, + option_int = 5, + option_real = 3.1, + option_enum = 'two', + option_string_val = NULL, + option_string_null = 'val'); +NOTICE: new option value for string parameter null +NOTICE: new option value for string parameter val +-- Silence again validation checks for strings until the end of the test. +SET client_min_messages TO 'warning'; +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +------------------------ + option_bool=false + option_int=5 + option_real=3.1 + option_enum=two + option_string_val=null + option_string_null=val +(6 rows) + +-- ALTER TABLE .. SET +ALTER TABLE dummy_test_tab SET (option_int = 10); +ALTER TABLE dummy_test_tab SET (option_bool = true); +ALTER TABLE dummy_test_tab SET (option_real = 3.2); +ALTER TABLE dummy_test_tab SET (option_string_val = 'val2'); +ALTER TABLE dummy_test_tab SET (option_string_null = NULL); +ALTER TABLE dummy_test_tab SET (option_enum = 'one'); +ALTER TABLE dummy_test_tab SET (option_enum = 'three'); +ERROR: invalid value for enum option "option_enum": three +DETAIL: Valid values are "one" and "two". +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +------------------------- + option_int=10 + option_bool=true + option_real=3.2 + option_string_val=val2 + option_string_null=null + option_enum=one +(6 rows) + +-- ALTER TABLE .. RESET +ALTER TABLE dummy_test_tab RESET (option_int); +ALTER TABLE dummy_test_tab RESET (option_bool); +ALTER TABLE dummy_test_tab RESET (option_real); +ALTER TABLE dummy_test_tab RESET (option_enum); +ALTER TABLE dummy_test_tab RESET (option_string_val); +ALTER TABLE dummy_test_tab RESET (option_string_null); +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +-------- +(0 rows) + +-- Cross-type checks for reloption values +-- Integer +ALTER TABLE dummy_test_tab SET (option_int = 3.3); -- ok +ALTER TABLE dummy_test_tab SET (option_int = true); -- error +ERROR: invalid value for integer option "option_int": true +ALTER TABLE dummy_test_tab SET (option_int = 'val3'); -- error +ERROR: invalid value for integer option "option_int": val3 +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +---------------- + option_int=3.3 +(1 row) + +ALTER TABLE dummy_test_tab RESET (option_int); +-- Boolean +ALTER TABLE dummy_test_tab SET (option_bool = 4); -- error +ERROR: invalid value for boolean option "option_bool": 4 +ALTER TABLE dummy_test_tab SET (option_bool = 1); -- ok, as true +ALTER TABLE dummy_test_tab SET (option_bool = 3.4); -- error +ERROR: invalid value for boolean option "option_bool": 3.4 +ALTER TABLE dummy_test_tab SET (option_bool = 'val4'); -- error +ERROR: invalid value for boolean option "option_bool": val4 +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +--------------- + option_bool=1 +(1 row) + +ALTER TABLE dummy_test_tab RESET (option_bool); +-- Float +ALTER TABLE dummy_test_tab SET (option_real = 4); -- ok +ALTER TABLE dummy_test_tab SET (option_real = true); -- error +ERROR: invalid value for floating point option "option_real": true +ALTER TABLE dummy_test_tab SET (option_real = 'val5'); -- error +ERROR: invalid value for floating point option "option_real": val5 +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +--------------- + option_real=4 +(1 row) + +ALTER TABLE dummy_test_tab RESET (option_real); +-- Enum +ALTER TABLE dummy_test_tab SET (option_enum = 'one'); -- ok +ALTER TABLE dummy_test_tab SET (option_enum = 0); -- error +ERROR: invalid value for enum option "option_enum": 0 +DETAIL: Valid values are "one" and "two". +ALTER TABLE dummy_test_tab SET (option_enum = true); -- error +ERROR: invalid value for enum option "option_enum": true +DETAIL: Valid values are "one" and "two". +ALTER TABLE dummy_test_tab SET (option_enum = 'three'); -- error +ERROR: invalid value for enum option "option_enum": three +DETAIL: Valid values are "one" and "two". +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +----------------- + option_enum=one +(1 row) + +ALTER TABLE dummy_test_tab RESET (option_enum); +-- String +ALTER TABLE dummy_test_tab SET (option_string_val = 4); -- ok +ALTER TABLE dummy_test_tab SET (option_string_val = 3.5); -- ok +ALTER TABLE dummy_test_tab SET (option_string_val = true); -- ok, as "true" +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + unnest +------------------------ + option_string_val=true +(1 row) + +ALTER TABLE dummy_test_tab RESET (option_string_val); +DROP TABLE dummy_test_tab; +-- ALTER TABLE SET ACCESS METHOD OPTIONS +CREATE TABLE heap_tab (i INT4) WITH (fillfactor=100, toast_tuple_target=1000); +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'heap_tab'; + unnest +------------------------- + fillfactor=100 + toast_tuple_target=1000 +(2 rows) + +-- error: fillfactor is out of bounds: maximum value from the new table am is 90 +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am; +ERROR: value 100 out of bounds for option "fillfactor" +DETAIL: Valid values are between "0" and "90". +-- error: toast_tuple_target does not exist in the new table AM +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (SET fillfactor '50'); +ERROR: unrecognized parameter "toast_tuple_target" +-- error: adding is not possible when the parameter is already defined in source reloptions +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (ADD fillfactor '50'); +ERROR: option "fillfactor" provided more than once +-- error: the specified option we want to drop does not exist +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (DROP does_not_exist); +ERROR: option "does_not_exist" not found +-- error: adding unrecognized parameter +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (SET fillfactor '50', DROP toast_tuple_target, ADD unrecognized 'foo'); +ERROR: unrecognized parameter "unrecognized" +-- ok +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (DROP fillfactor, DROP toast_tuple_target, option_int '1', option_bool 'true', option_real '0.001', option_enum 'one', option_string_val 'hello'); +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'heap_tab'; + unnest +------------------------- + option_int=1 + option_bool=true + option_real=0.001 + option_enum=one + option_string_val=hello +(5 rows) + +DROP TABLE heap_tab; diff --git a/src/test/modules/dummy_table_am/meson.build b/src/test/modules/dummy_table_am/meson.build new file mode 100644 index 00000000000..6b197b15ffa --- /dev/null +++ b/src/test/modules/dummy_table_am/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2022-2025, PostgreSQL Global Development Group + +dummy_table_am_sources = files( + 'dummy_table_am.c', +) + +if host_system == 'windows' + dummy_table_am_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'dummy_table_am', + '--FILEDESC', 'dummy_table_am - table access method template',]) +endif + +dummy_table_am = shared_module('dummy_table_am', + dummy_table_am_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += dummy_table_am + +test_install_data += files( + 'dummy_table_am.control', + 'dummy_table_am--1.0.sql', +) + +tests += { + 'name': 'dummy_table_am', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'reloptions', + ], + }, +} diff --git a/src/test/modules/dummy_table_am/sql/reloptions.sql b/src/test/modules/dummy_table_am/sql/reloptions.sql new file mode 100644 index 00000000000..47fb4862c6c --- /dev/null +++ b/src/test/modules/dummy_table_am/sql/reloptions.sql @@ -0,0 +1,99 @@ +-- Tests for relation options +CREATE EXTENSION dummy_table_am; + +CREATE TABLE dummy_test_tab (i int4) USING dummy_table_am; + +-- Silence validation checks for strings +SET client_min_messages TO 'warning'; + +-- Test with default values. +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; +DROP TABLE dummy_test_tab; + +-- Test with full set of options. +-- Allow validation checks for strings +SET client_min_messages TO 'notice'; +CREATE TABLE dummy_test_tab (i int4) + USING dummy_table_am WITH ( + option_bool = false, + option_int = 5, + option_real = 3.1, + option_enum = 'two', + option_string_val = NULL, + option_string_null = 'val'); +-- Silence again validation checks for strings until the end of the test. +SET client_min_messages TO 'warning'; +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + +-- ALTER TABLE .. SET +ALTER TABLE dummy_test_tab SET (option_int = 10); +ALTER TABLE dummy_test_tab SET (option_bool = true); +ALTER TABLE dummy_test_tab SET (option_real = 3.2); +ALTER TABLE dummy_test_tab SET (option_string_val = 'val2'); +ALTER TABLE dummy_test_tab SET (option_string_null = NULL); +ALTER TABLE dummy_test_tab SET (option_enum = 'one'); +ALTER TABLE dummy_test_tab SET (option_enum = 'three'); +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + +-- ALTER TABLE .. RESET +ALTER TABLE dummy_test_tab RESET (option_int); +ALTER TABLE dummy_test_tab RESET (option_bool); +ALTER TABLE dummy_test_tab RESET (option_real); +ALTER TABLE dummy_test_tab RESET (option_enum); +ALTER TABLE dummy_test_tab RESET (option_string_val); +ALTER TABLE dummy_test_tab RESET (option_string_null); +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; + +-- Cross-type checks for reloption values +-- Integer +ALTER TABLE dummy_test_tab SET (option_int = 3.3); -- ok +ALTER TABLE dummy_test_tab SET (option_int = true); -- error +ALTER TABLE dummy_test_tab SET (option_int = 'val3'); -- error +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; +ALTER TABLE dummy_test_tab RESET (option_int); +-- Boolean +ALTER TABLE dummy_test_tab SET (option_bool = 4); -- error +ALTER TABLE dummy_test_tab SET (option_bool = 1); -- ok, as true +ALTER TABLE dummy_test_tab SET (option_bool = 3.4); -- error +ALTER TABLE dummy_test_tab SET (option_bool = 'val4'); -- error +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; +ALTER TABLE dummy_test_tab RESET (option_bool); +-- Float +ALTER TABLE dummy_test_tab SET (option_real = 4); -- ok +ALTER TABLE dummy_test_tab SET (option_real = true); -- error +ALTER TABLE dummy_test_tab SET (option_real = 'val5'); -- error +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; +ALTER TABLE dummy_test_tab RESET (option_real); +-- Enum +ALTER TABLE dummy_test_tab SET (option_enum = 'one'); -- ok +ALTER TABLE dummy_test_tab SET (option_enum = 0); -- error +ALTER TABLE dummy_test_tab SET (option_enum = true); -- error +ALTER TABLE dummy_test_tab SET (option_enum = 'three'); -- error +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; +ALTER TABLE dummy_test_tab RESET (option_enum); +-- String +ALTER TABLE dummy_test_tab SET (option_string_val = 4); -- ok +ALTER TABLE dummy_test_tab SET (option_string_val = 3.5); -- ok +ALTER TABLE dummy_test_tab SET (option_string_val = true); -- ok, as "true" +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'dummy_test_tab'; +ALTER TABLE dummy_test_tab RESET (option_string_val); + +DROP TABLE dummy_test_tab; + +-- ALTER TABLE SET ACCESS METHOD OPTIONS +CREATE TABLE heap_tab (i INT4) WITH (fillfactor=100, toast_tuple_target=1000); +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'heap_tab'; +-- error: fillfactor is out of bounds: maximum value from the new table am is 90 +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am; +-- error: toast_tuple_target does not exist in the new table AM +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (SET fillfactor '50'); +-- error: adding is not possible when the parameter is already defined in source reloptions +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (ADD fillfactor '50'); +-- error: the specified option we want to drop does not exist +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (DROP does_not_exist); +-- error: adding unrecognized parameter +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (SET fillfactor '50', DROP toast_tuple_target, ADD unrecognized 'foo'); +-- ok +ALTER TABLE heap_tab SET ACCESS METHOD dummy_table_am OPTIONS (DROP fillfactor, DROP toast_tuple_target, option_int '1', option_bool 'true', option_real '0.001', option_enum 'one', option_string_val 'hello'); +SELECT unnest(reloptions) FROM pg_class WHERE relname = 'heap_tab'; +DROP TABLE heap_tab; diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 2b057451473..28398254df7 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -4,6 +4,7 @@ subdir('brin') subdir('commit_ts') subdir('delay_execution') subdir('dummy_index_am') +subdir('dummy_table_am') subdir('dummy_seclabel') subdir('gin') subdir('injection_points') -- 2.39.5 --2wd4swaawatv6v36-- ^ permalink raw reply [nested|flat] 157+ messages in thread
end of thread, other threads:[~2025-03-01 19:50 UTC | newest] Thread overview: 157+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2025-03-01 19:50 [PATCH 2/2] Add the "dummy_table_am" test module Julien Tachoires <julien@tachoires.me>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox