agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. 157+ messages / 2 participants [nested] [flat]
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) The utility of this is to reindex N partitions in 2 transactions, rather than 2*N transactions. --- src/backend/commands/indexcmds.c | 258 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 182 insertions(+), 80 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b21555b6cf..f5fea14ff4 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_indexes(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_indexes(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_indexes(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + ReindexParams *params, MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 830fdddf24..6f41adf736 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once @ 2020-11-01 18:25 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 157+ messages in thread From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw) --- src/backend/commands/indexcmds.c | 262 ++++++++++++++------- src/test/regress/expected/create_index.out | 4 +- 2 files changed, 185 insertions(+), 81 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 739bd14001..08d44a1999 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids, ReindexParams *params); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); +static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, + int options, MemoryContext private_context); static void update_relispartition(Oid relationId, bool newval); static inline void set_indexsafe_procflags(void); @@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; +/* Argument to ReindexIndexConcurrently takes a List* of these */ +typedef struct ReindexIndexInfo +{ + Oid indexId; + Oid tableId; + Oid amId; + bool safe; /* for set_indexsafe_procflags */ +} ReindexIndexInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId, static void reindex_invalid_child_indexes(Oid indexRelationId) { - ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID, + }; /* * CIC needs to mark a partitioned index as VALID, which itself @@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel) ReindexPartitions(indOid, params, isTopLevel); else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && persistence != RELPERSISTENCE_TEMP) - ReindexRelationConcurrently(indOid, params); + { + ReindexIndexInfo idxinfo = { + .indexId = indOid, + /* other fields set later */ + }; + ReindexIndexesConcurrently(list_make1(&idxinfo), + list_make1_oid(IndexGetRelation(indOid, false)), + params->options, CurrentMemoryContext); + } else { ReindexParams newparams = *params; @@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg) errinfo->relnamespace, errinfo->relname); } + +/* + * Given a list of index oids, return a list of leaf partitions by removing + * any intermediate parents. heaprels is populated with the corresponding + * tables. + */ +static List * +leaf_partitions(List *inhoids, int options, List **heaprels) +{ + List *partitions = NIL; + ListCell *lc; + + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid tableoid; + Relation table; + char partkind = get_rel_relkind(partoid); + + /* + * This discards partitioned indexes and foreign tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX); + + /* Skip invalid indexes, if requested */ + if ((options & REINDEXOPT_SKIPVALID) != 0 && + get_index_isvalid(partoid)) + continue; + + /* (try to) Open the table, with lock */ + tableoid = IndexGetRelation(partoid, false); + table = table_open(tableoid, ShareLock); + table_close(table, NoLock); + + /* Save partition OID in current MemoryContext */ + partitions = lappend_oid(partitions, partoid); + *heaprels = lappend_oid(*heaprels, tableoid); + } + + return partitions; +} + + /* * ReindexPartitions * * Reindex a set of partitions, per the partitioned index or table given * by the caller. + * XXX: should be further refactored with logic from ReindexRelationConcurrently */ static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) { - List *partitions = NIL; + List *partitions = NIL, + *heaprels = NIL; char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); MemoryContext reindex_context; + MemoryContext old_context; List *inhoids; ListCell *lc; ErrorContextCallback errcallback; @@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * The list of relations to reindex are the physical partitions of the * tree so discard any partitioned table or index. */ - foreach(lc, inhoids) - { - Oid partoid = lfirst_oid(lc); - char partkind = get_rel_relkind(partoid); - MemoryContext old_context; - /* - * This discards partitioned tables, partitioned indexes and foreign - * tables. - */ - if (!RELKIND_HAS_STORAGE(partkind)) - continue; - - /* Skip invalid indexes, if requested */ - if ((params->options & REINDEXOPT_SKIPVALID) != 0 && - get_index_isvalid(partoid)) - continue; + if (relkind == RELKIND_PARTITIONED_INDEX) + { + old_context = MemoryContextSwitchTo(reindex_context); + partitions = leaf_partitions(inhoids, params->options, &heaprels); + MemoryContextSwitchTo(old_context); + } else { + /* Loop over parent tables */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Relation parttable; + List *partindexes; + + parttable = table_open(partoid, ShareLock); + old_context = MemoryContextSwitchTo(reindex_context); + partindexes = RelationGetIndexList(parttable); + partindexes = leaf_partitions(partindexes, params->options, &heaprels); + partitions = list_concat(partitions, partindexes); + + MemoryContextSwitchTo(old_context); + table_close(parttable, ShareLock); + } + } - Assert(partkind == RELKIND_INDEX || - partkind == RELKIND_RELATION); + if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 && + relkind == RELKIND_PARTITIONED_INDEX && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + { + List *idxinfos = NIL; + ReindexIndexInfo *idxinfo; - /* Save partition OID */ old_context = MemoryContextSwitchTo(reindex_context); - partitions = lappend_oid(partitions, partoid); + foreach (lc, partitions) + { + Oid partoid = lfirst_oid(lc); + idxinfo = palloc(sizeof(ReindexIndexInfo)); + idxinfo->indexId = partoid; + /* other fields set later */ + idxinfos = lappend(idxinfos, idxinfo); + } MemoryContextSwitchTo(old_context); - } - /* - * Process each partition listed in a separate transaction. Note that - * this commits and then starts a new transaction immediately. - */ - ReindexMultipleInternal(partitions, params); + /* Process all indexes in a single loop */ + ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context); + } else { + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, params); + } /* * If indexes exist on all of the partitioned table's children, and we @@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) { - typedef struct ReindexIndexInfo - { - Oid indexId; - Oid tableId; - Oid amId; - bool safe; /* for set_indexsafe_procflags */ - } ReindexIndexInfo; List *heapRelationIds = NIL; List *indexIds = NIL; List *newIndexIds = NIL; - List *relationLocks = NIL; - List *lockTags = NIL; ListCell *lc, *lc2; MemoryContext private_context; @@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; - const int progress_index[] = { - PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_INDEX_OID, - PROGRESS_CREATEIDX_ACCESS_METHOD_OID - }; - int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) Assert(heapRelationIds != NIL); + /* Do the work */ + newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context); + + /* Log what we did */ + if ((params->options & REINDEXOPT_VERBOSE) != 0) + { + if (relkind == RELKIND_INDEX) + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + else + { + foreach(lc, newIndexIds) + { + Oid indOid = lfirst_oid(lc); + ereport(INFO, + (errmsg("index \"%s.%s\" was reindexed", + get_namespace_name(get_rel_namespace(indOid)), + get_rel_name(indOid)))); + /* Don't show rusage here, since it's not per index. */ + } + + ereport(INFO, + (errmsg("table \"%s.%s\" was reindexed", + relationNamespace, relationName), + errdetail("%s.", + pg_rusage_show(&ru0)))); + } + } + + + MemoryContextDelete(private_context); + + return true; +} + +/* + * Reindex concurrently for an arbitrary list of index relations + * This is called by ReindexRelationConcurrently and + */ +static List * +ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options, + MemoryContext private_context) +{ + List *newIndexIds = NIL; + List *relationLocks = NIL; + List *lockTags = NIL; + + ListCell *lc, + *lc2; + + MemoryContext oldcontext; + + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; + /*----- * Now we have all the indexes we want to process in indexIds. * @@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* Start a new transaction to finish process properly */ StartTransactionCommand(); - /* Log what we did */ - if ((params->options & REINDEXOPT_VERBOSE) != 0) - { - if (relkind == RELKIND_INDEX) - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - else - { - foreach(lc, newIndexIds) - { - ReindexIndexInfo *idx = lfirst(lc); - Oid indOid = idx->indexId; - - ereport(INFO, - (errmsg("index \"%s.%s\" was reindexed", - get_namespace_name(get_rel_namespace(indOid)), - get_rel_name(indOid)))); - /* Don't show rusage here, since it's not per index. */ - } - - ereport(INFO, - (errmsg("table \"%s.%s\" was reindexed", - relationNamespace, relationName), - errdetail("%s.", - pg_rusage_show(&ru0)))); - } - } - - MemoryContextDelete(private_context); - pgstat_progress_end_command(); - return true; + return newIndexIds; } /* diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index fc6afab58a..4a03ab2abb 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2470,12 +2470,12 @@ COMMIT; REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported -- These are the toast table and index of pg_authid. REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently +ERROR: concurrent index creation on system catalog tables is not supported REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM ERROR: cannot reindex system catalogs concurrently -- Warns about catalog relations -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
* [PATCH v11 6/7] Row pattern recognition patch (tests). @ 2023-11-08 06:57 Tatsuo Ishii <ishii@postgresql.org> 0 siblings, 0 replies; 157+ messages in thread From: Tatsuo Ishii @ 2023-11-08 06:57 UTC (permalink / raw) --- src/test/regress/expected/rpr.out | 821 +++++++++++++++++++++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/rpr.sql | 392 ++++++++++++++ 3 files changed, 1214 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/rpr.out create mode 100644 src/test/regress/sql/rpr.sql diff --git a/src/test/regress/expected/rpr.out b/src/test/regress/expected/rpr.out new file mode 100644 index 0000000000..e8998ebf45 --- /dev/null +++ b/src/test/regress/expected/rpr.out @@ -0,0 +1,821 @@ +-- +-- Test for row pattern definition clause +-- +CREATE TEMP TABLE stock ( + company TEXT, + tdate DATE, + price INTEGER +); +INSERT INTO stock VALUES ('company1', '2023-07-01', 100); +INSERT INTO stock VALUES ('company1', '2023-07-02', 200); +INSERT INTO stock VALUES ('company1', '2023-07-03', 150); +INSERT INTO stock VALUES ('company1', '2023-07-04', 140); +INSERT INTO stock VALUES ('company1', '2023-07-05', 150); +INSERT INTO stock VALUES ('company1', '2023-07-06', 90); +INSERT INTO stock VALUES ('company1', '2023-07-07', 110); +INSERT INTO stock VALUES ('company1', '2023-07-08', 130); +INSERT INTO stock VALUES ('company1', '2023-07-09', 120); +INSERT INTO stock VALUES ('company1', '2023-07-10', 130); +INSERT INTO stock VALUES ('company2', '2023-07-01', 50); +INSERT INTO stock VALUES ('company2', '2023-07-02', 2000); +INSERT INTO stock VALUES ('company2', '2023-07-03', 1500); +INSERT INTO stock VALUES ('company2', '2023-07-04', 1400); +INSERT INTO stock VALUES ('company2', '2023-07-05', 1500); +INSERT INTO stock VALUES ('company2', '2023-07-06', 60); +INSERT INTO stock VALUES ('company2', '2023-07-07', 1100); +INSERT INTO stock VALUES ('company2', '2023-07-08', 1300); +INSERT INTO stock VALUES ('company2', '2023-07-09', 1200); +INSERT INTO stock VALUES ('company2', '2023-07-10', 1300); +SELECT * FROM stock; + company | tdate | price +----------+------------+------- + company1 | 07-01-2023 | 100 + company1 | 07-02-2023 | 200 + company1 | 07-03-2023 | 150 + company1 | 07-04-2023 | 140 + company1 | 07-05-2023 | 150 + company1 | 07-06-2023 | 90 + company1 | 07-07-2023 | 110 + company1 | 07-08-2023 | 130 + company1 | 07-09-2023 | 120 + company1 | 07-10-2023 | 130 + company2 | 07-01-2023 | 50 + company2 | 07-02-2023 | 2000 + company2 | 07-03-2023 | 1500 + company2 | 07-04-2023 | 1400 + company2 | 07-05-2023 | 1500 + company2 | 07-06-2023 | 60 + company2 | 07-07-2023 | 1100 + company2 | 07-08-2023 | 1300 + company2 | 07-09-2023 | 1200 + company2 | 07-10-2023 | 1300 +(20 rows) + +-- basic test using PREV +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value | nth_second +----------+------------+-------+-------------+------------+------------ + company1 | 07-01-2023 | 100 | 100 | 140 | 07-02-2023 + company1 | 07-02-2023 | 200 | | | + company1 | 07-03-2023 | 150 | | | + company1 | 07-04-2023 | 140 | | | + company1 | 07-05-2023 | 150 | | | + company1 | 07-06-2023 | 90 | 90 | 120 | 07-07-2023 + company1 | 07-07-2023 | 110 | | | + company1 | 07-08-2023 | 130 | | | + company1 | 07-09-2023 | 120 | | | + company1 | 07-10-2023 | 130 | | | + company2 | 07-01-2023 | 50 | 50 | 1400 | 07-02-2023 + company2 | 07-02-2023 | 2000 | | | + company2 | 07-03-2023 | 1500 | | | + company2 | 07-04-2023 | 1400 | | | + company2 | 07-05-2023 | 1500 | | | + company2 | 07-06-2023 | 60 | 60 | 1200 | 07-07-2023 + company2 | 07-07-2023 | 1100 | | | + company2 | 07-08-2023 | 1300 | | | + company2 | 07-09-2023 | 1200 | | | + company2 | 07-10-2023 | 1300 | | | +(20 rows) + +-- basic test using PREV. UP appears twice +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+ UP+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value | nth_second +----------+------------+-------+-------------+------------+------------ + company1 | 07-01-2023 | 100 | 100 | 150 | 07-02-2023 + company1 | 07-02-2023 | 200 | | | + company1 | 07-03-2023 | 150 | | | + company1 | 07-04-2023 | 140 | | | + company1 | 07-05-2023 | 150 | | | + company1 | 07-06-2023 | 90 | 90 | 130 | 07-07-2023 + company1 | 07-07-2023 | 110 | | | + company1 | 07-08-2023 | 130 | | | + company1 | 07-09-2023 | 120 | | | + company1 | 07-10-2023 | 130 | | | + company2 | 07-01-2023 | 50 | 50 | 1500 | 07-02-2023 + company2 | 07-02-2023 | 2000 | | | + company2 | 07-03-2023 | 1500 | | | + company2 | 07-04-2023 | 1400 | | | + company2 | 07-05-2023 | 1500 | | | + company2 | 07-06-2023 | 60 | 60 | 1300 | 07-07-2023 + company2 | 07-07-2023 | 1100 | | | + company2 | 07-08-2023 | 1300 | | | + company2 | 07-09-2023 | 1200 | | | + company2 | 07-10-2023 | 1300 | | | +(20 rows) + +-- basic test using PREV. Use '*' +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP* DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value | nth_second +----------+------------+-------+-------------+------------+------------ + company1 | 07-01-2023 | 100 | 100 | 140 | 07-02-2023 + company1 | 07-02-2023 | 200 | | | + company1 | 07-03-2023 | 150 | | | + company1 | 07-04-2023 | 140 | | | + company1 | 07-05-2023 | 150 | 150 | 90 | 07-06-2023 + company1 | 07-06-2023 | 90 | | | + company1 | 07-07-2023 | 110 | 110 | 120 | 07-08-2023 + company1 | 07-08-2023 | 130 | | | + company1 | 07-09-2023 | 120 | | | + company1 | 07-10-2023 | 130 | | | + company2 | 07-01-2023 | 50 | 50 | 1400 | 07-02-2023 + company2 | 07-02-2023 | 2000 | | | + company2 | 07-03-2023 | 1500 | | | + company2 | 07-04-2023 | 1400 | | | + company2 | 07-05-2023 | 1500 | 1500 | 60 | 07-06-2023 + company2 | 07-06-2023 | 60 | | | + company2 | 07-07-2023 | 1100 | 1100 | 1200 | 07-08-2023 + company2 | 07-08-2023 | 1300 | | | + company2 | 07-09-2023 | 1200 | | | + company2 | 07-10-2023 | 1300 | | | +(20 rows) + +-- basic test with none greedy pattern +SELECT company, tdate, price, count(*) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (A A A) + DEFINE + A AS price >= 140 AND price <= 150 +); + company | tdate | price | count +----------+------------+-------+------- + company1 | 07-01-2023 | 100 | 0 + company1 | 07-02-2023 | 200 | 0 + company1 | 07-03-2023 | 150 | 3 + company1 | 07-04-2023 | 140 | + company1 | 07-05-2023 | 150 | + company1 | 07-06-2023 | 90 | 0 + company1 | 07-07-2023 | 110 | 0 + company1 | 07-08-2023 | 130 | 0 + company1 | 07-09-2023 | 120 | 0 + company1 | 07-10-2023 | 130 | 0 + company2 | 07-01-2023 | 50 | 0 + company2 | 07-02-2023 | 2000 | 0 + company2 | 07-03-2023 | 1500 | 0 + company2 | 07-04-2023 | 1400 | 0 + company2 | 07-05-2023 | 1500 | 0 + company2 | 07-06-2023 | 60 | 0 + company2 | 07-07-2023 | 1100 | 0 + company2 | 07-08-2023 | 1300 | 0 + company2 | 07-09-2023 | 1200 | 0 + company2 | 07-10-2023 | 1300 | 0 +(20 rows) + +-- last_value() should remain consistent +SELECT company, tdate, price, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + company | tdate | price | last_value +----------+------------+-------+------------ + company1 | 07-01-2023 | 100 | 140 + company1 | 07-02-2023 | 200 | + company1 | 07-03-2023 | 150 | + company1 | 07-04-2023 | 140 | + company1 | 07-05-2023 | 150 | + company1 | 07-06-2023 | 90 | 120 + company1 | 07-07-2023 | 110 | + company1 | 07-08-2023 | 130 | + company1 | 07-09-2023 | 120 | + company1 | 07-10-2023 | 130 | + company2 | 07-01-2023 | 50 | 1400 + company2 | 07-02-2023 | 2000 | + company2 | 07-03-2023 | 1500 | + company2 | 07-04-2023 | 1400 | + company2 | 07-05-2023 | 1500 | + company2 | 07-06-2023 | 60 | 1200 + company2 | 07-07-2023 | 1100 | + company2 | 07-08-2023 | 1300 | + company2 | 07-09-2023 | 1200 | + company2 | 07-10-2023 | 1300 | +(20 rows) + +-- omit "START" in DEFINE but it is ok because "START AS TRUE" is +-- implicitly defined. per spec. +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value | nth_second +----------+------------+-------+-------------+------------+------------ + company1 | 07-01-2023 | 100 | 100 | 140 | 07-02-2023 + company1 | 07-02-2023 | 200 | | | + company1 | 07-03-2023 | 150 | | | + company1 | 07-04-2023 | 140 | | | + company1 | 07-05-2023 | 150 | | | + company1 | 07-06-2023 | 90 | 90 | 120 | 07-07-2023 + company1 | 07-07-2023 | 110 | | | + company1 | 07-08-2023 | 130 | | | + company1 | 07-09-2023 | 120 | | | + company1 | 07-10-2023 | 130 | | | + company2 | 07-01-2023 | 50 | 50 | 1400 | 07-02-2023 + company2 | 07-02-2023 | 2000 | | | + company2 | 07-03-2023 | 1500 | | | + company2 | 07-04-2023 | 1400 | | | + company2 | 07-05-2023 | 1500 | | | + company2 | 07-06-2023 | 60 | 60 | 1200 | 07-07-2023 + company2 | 07-07-2023 | 1100 | | | + company2 | 07-08-2023 | 1300 | | | + company2 | 07-09-2023 | 1200 | | | + company2 | 07-10-2023 | 1300 | | | +(20 rows) + +-- the first row start with less than or equal to 100 +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (LOWPRICE UP+ DOWN+) + DEFINE + LOWPRICE AS price <= 100, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value +----------+------------+-------+-------------+------------ + company1 | 07-01-2023 | 100 | 100 | 140 + company1 | 07-02-2023 | 200 | | + company1 | 07-03-2023 | 150 | | + company1 | 07-04-2023 | 140 | | + company1 | 07-05-2023 | 150 | | + company1 | 07-06-2023 | 90 | 90 | 120 + company1 | 07-07-2023 | 110 | | + company1 | 07-08-2023 | 130 | | + company1 | 07-09-2023 | 120 | | + company1 | 07-10-2023 | 130 | | + company2 | 07-01-2023 | 50 | 50 | 1400 + company2 | 07-02-2023 | 2000 | | + company2 | 07-03-2023 | 1500 | | + company2 | 07-04-2023 | 1400 | | + company2 | 07-05-2023 | 1500 | | + company2 | 07-06-2023 | 60 | 60 | 1200 + company2 | 07-07-2023 | 1100 | | + company2 | 07-08-2023 | 1300 | | + company2 | 07-09-2023 | 1200 | | + company2 | 07-10-2023 | 1300 | | +(20 rows) + +-- second row raises 120% +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (LOWPRICE UP+ DOWN+) + DEFINE + LOWPRICE AS price <= 100, + UP AS price > PREV(price) * 1.2, + DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value +----------+------------+-------+-------------+------------ + company1 | 07-01-2023 | 100 | 100 | 140 + company1 | 07-02-2023 | 200 | | + company1 | 07-03-2023 | 150 | | + company1 | 07-04-2023 | 140 | | + company1 | 07-05-2023 | 150 | | + company1 | 07-06-2023 | 90 | | + company1 | 07-07-2023 | 110 | | + company1 | 07-08-2023 | 130 | | + company1 | 07-09-2023 | 120 | | + company1 | 07-10-2023 | 130 | | + company2 | 07-01-2023 | 50 | 50 | 1400 + company2 | 07-02-2023 | 2000 | | + company2 | 07-03-2023 | 1500 | | + company2 | 07-04-2023 | 1400 | | + company2 | 07-05-2023 | 1500 | | + company2 | 07-06-2023 | 60 | | + company2 | 07-07-2023 | 1100 | | + company2 | 07-08-2023 | 1300 | | + company2 | 07-09-2023 | 1200 | | + company2 | 07-10-2023 | 1300 | | +(20 rows) + +-- using NEXT +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UPDOWN) + DEFINE + START AS TRUE, + UPDOWN AS price > PREV(price) AND price > NEXT(price) +); + company | tdate | price | first_value | last_value +----------+------------+-------+-------------+------------ + company1 | 07-01-2023 | 100 | 100 | 200 + company1 | 07-02-2023 | 200 | | + company1 | 07-03-2023 | 150 | | + company1 | 07-04-2023 | 140 | 140 | 150 + company1 | 07-05-2023 | 150 | | + company1 | 07-06-2023 | 90 | | + company1 | 07-07-2023 | 110 | 110 | 130 + company1 | 07-08-2023 | 130 | | + company1 | 07-09-2023 | 120 | | + company1 | 07-10-2023 | 130 | | + company2 | 07-01-2023 | 50 | 50 | 2000 + company2 | 07-02-2023 | 2000 | | + company2 | 07-03-2023 | 1500 | | + company2 | 07-04-2023 | 1400 | 1400 | 1500 + company2 | 07-05-2023 | 1500 | | + company2 | 07-06-2023 | 60 | | + company2 | 07-07-2023 | 1100 | 1100 | 1300 + company2 | 07-08-2023 | 1300 | | + company2 | 07-09-2023 | 1200 | | + company2 | 07-10-2023 | 1300 | | +(20 rows) + +-- using AFTER MATCH SKIP TO NEXT ROW +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP TO NEXT ROW + INITIAL + PATTERN (START UPDOWN) + DEFINE + START AS TRUE, + UPDOWN AS price > PREV(price) AND price > NEXT(price) +); + company | tdate | price | first_value | last_value +----------+------------+-------+-------------+------------ + company1 | 07-01-2023 | 100 | 100 | 200 + company1 | 07-02-2023 | 200 | | + company1 | 07-03-2023 | 150 | | + company1 | 07-04-2023 | 140 | 140 | 150 + company1 | 07-05-2023 | 150 | | + company1 | 07-06-2023 | 90 | | + company1 | 07-07-2023 | 110 | 110 | 130 + company1 | 07-08-2023 | 130 | | + company1 | 07-09-2023 | 120 | | + company1 | 07-10-2023 | 130 | | + company2 | 07-01-2023 | 50 | 50 | 2000 + company2 | 07-02-2023 | 2000 | | + company2 | 07-03-2023 | 1500 | | + company2 | 07-04-2023 | 1400 | 1400 | 1500 + company2 | 07-05-2023 | 1500 | | + company2 | 07-06-2023 | 60 | | + company2 | 07-07-2023 | 1100 | 1100 | 1300 + company2 | 07-08-2023 | 1300 | | + company2 | 07-09-2023 | 1200 | | + company2 | 07-10-2023 | 1300 | | +(20 rows) + +-- match everything +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP PAST LAST ROW + INITIAL + PATTERN (A+) + DEFINE + A AS TRUE +); + company | tdate | price | first_value | last_value +----------+------------+-------+-------------+------------ + company1 | 07-01-2023 | 100 | 100 | 130 + company1 | 07-02-2023 | 200 | | + company1 | 07-03-2023 | 150 | | + company1 | 07-04-2023 | 140 | | + company1 | 07-05-2023 | 150 | | + company1 | 07-06-2023 | 90 | | + company1 | 07-07-2023 | 110 | | + company1 | 07-08-2023 | 130 | | + company1 | 07-09-2023 | 120 | | + company1 | 07-10-2023 | 130 | | + company2 | 07-01-2023 | 50 | 50 | 1300 + company2 | 07-02-2023 | 2000 | | + company2 | 07-03-2023 | 1500 | | + company2 | 07-04-2023 | 1400 | | + company2 | 07-05-2023 | 1500 | | + company2 | 07-06-2023 | 60 | | + company2 | 07-07-2023 | 1100 | | + company2 | 07-08-2023 | 1300 | | + company2 | 07-09-2023 | 1200 | | + company2 | 07-10-2023 | 1300 | | +(20 rows) + +-- backtracking with reclassification of rows +-- using AFTER MATCH SKIP PAST LAST ROW +SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP PAST LAST ROW + INITIAL + PATTERN (A+ B+) + DEFINE + A AS price > 100, + B AS price > 100 +); + company | tdate | price | first_value | last_value +----------+------------+-------+-------------+------------ + company1 | 07-01-2023 | 100 | | + company1 | 07-02-2023 | 200 | 07-02-2023 | 07-05-2023 + company1 | 07-03-2023 | 150 | | + company1 | 07-04-2023 | 140 | | + company1 | 07-05-2023 | 150 | | + company1 | 07-06-2023 | 90 | | + company1 | 07-07-2023 | 110 | 07-07-2023 | 07-10-2023 + company1 | 07-08-2023 | 130 | | + company1 | 07-09-2023 | 120 | | + company1 | 07-10-2023 | 130 | | + company2 | 07-01-2023 | 50 | | + company2 | 07-02-2023 | 2000 | 07-02-2023 | 07-05-2023 + company2 | 07-03-2023 | 1500 | | + company2 | 07-04-2023 | 1400 | | + company2 | 07-05-2023 | 1500 | | + company2 | 07-06-2023 | 60 | | + company2 | 07-07-2023 | 1100 | 07-07-2023 | 07-10-2023 + company2 | 07-08-2023 | 1300 | | + company2 | 07-09-2023 | 1200 | | + company2 | 07-10-2023 | 1300 | | +(20 rows) + +-- backtracking with reclassification of rows +-- using AFTER MATCH SKIP TO NEXT ROW +SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP TO NEXT ROW + INITIAL + PATTERN (A+ B+) + DEFINE + A AS price > 100, + B AS price > 100 +); + company | tdate | price | first_value | last_value +----------+------------+-------+-------------+------------ + company1 | 07-01-2023 | 100 | | + company1 | 07-02-2023 | 200 | 07-02-2023 | 07-05-2023 + company1 | 07-03-2023 | 150 | 07-03-2023 | 07-05-2023 + company1 | 07-04-2023 | 140 | 07-04-2023 | 07-05-2023 + company1 | 07-05-2023 | 150 | | + company1 | 07-06-2023 | 90 | | + company1 | 07-07-2023 | 110 | 07-07-2023 | 07-10-2023 + company1 | 07-08-2023 | 130 | 07-08-2023 | 07-10-2023 + company1 | 07-09-2023 | 120 | 07-09-2023 | 07-10-2023 + company1 | 07-10-2023 | 130 | | + company2 | 07-01-2023 | 50 | | + company2 | 07-02-2023 | 2000 | 07-02-2023 | 07-05-2023 + company2 | 07-03-2023 | 1500 | 07-03-2023 | 07-05-2023 + company2 | 07-04-2023 | 1400 | 07-04-2023 | 07-05-2023 + company2 | 07-05-2023 | 1500 | | + company2 | 07-06-2023 | 60 | | + company2 | 07-07-2023 | 1100 | 07-07-2023 | 07-10-2023 + company2 | 07-08-2023 | 1300 | 07-08-2023 | 07-10-2023 + company2 | 07-09-2023 | 1200 | 07-09-2023 | 07-10-2023 + company2 | 07-10-2023 | 1300 | | +(20 rows) + +-- ROWS BETWEEN CURRENT ROW AND offset FOLLOWING +SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w, + count(*) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND 2 FOLLOWING + AFTER MATCH SKIP PAST LAST ROW + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value | count +----------+------------+-------+-------------+------------+------- + company1 | 07-01-2023 | 100 | 07-01-2023 | 07-03-2023 | 3 + company1 | 07-02-2023 | 200 | | | + company1 | 07-03-2023 | 150 | | | + company1 | 07-04-2023 | 140 | 07-04-2023 | 07-06-2023 | 3 + company1 | 07-05-2023 | 150 | | | + company1 | 07-06-2023 | 90 | | | + company1 | 07-07-2023 | 110 | 07-07-2023 | 07-09-2023 | 3 + company1 | 07-08-2023 | 130 | | | + company1 | 07-09-2023 | 120 | | | + company1 | 07-10-2023 | 130 | | | 0 + company2 | 07-01-2023 | 50 | 07-01-2023 | 07-03-2023 | 3 + company2 | 07-02-2023 | 2000 | | | + company2 | 07-03-2023 | 1500 | | | + company2 | 07-04-2023 | 1400 | 07-04-2023 | 07-06-2023 | 3 + company2 | 07-05-2023 | 1500 | | | + company2 | 07-06-2023 | 60 | | | + company2 | 07-07-2023 | 1100 | 07-07-2023 | 07-09-2023 | 3 + company2 | 07-08-2023 | 1300 | | | + company2 | 07-09-2023 | 1200 | | | + company2 | 07-10-2023 | 1300 | | | 0 +(20 rows) + +-- +-- Aggregates +-- +-- using AFTER MATCH SKIP PAST LAST ROW +SELECT company, tdate, price, + first_value(price) OVER w, + last_value(price) OVER w, + max(price) OVER w, + min(price) OVER w, + sum(price) OVER w, + avg(price) OVER w, + count(price) OVER w +FROM stock +WINDOW w AS ( +PARTITION BY company +ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +AFTER MATCH SKIP PAST LAST ROW +INITIAL +PATTERN (START UP+ DOWN+) +DEFINE +START AS TRUE, +UP AS price > PREV(price), +DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value | max | min | sum | avg | count +----------+------------+-------+-------------+------------+------+-----+------+-----------------------+------- + company1 | 07-01-2023 | 100 | 100 | 140 | 200 | 100 | 590 | 147.5000000000000000 | 4 + company1 | 07-02-2023 | 200 | | | | | | | + company1 | 07-03-2023 | 150 | | | | | | | + company1 | 07-04-2023 | 140 | | | | | | | + company1 | 07-05-2023 | 150 | | | | | | | 0 + company1 | 07-06-2023 | 90 | 90 | 120 | 130 | 90 | 450 | 112.5000000000000000 | 4 + company1 | 07-07-2023 | 110 | | | | | | | + company1 | 07-08-2023 | 130 | | | | | | | + company1 | 07-09-2023 | 120 | | | | | | | + company1 | 07-10-2023 | 130 | | | | | | | 0 + company2 | 07-01-2023 | 50 | 50 | 1400 | 2000 | 50 | 4950 | 1237.5000000000000000 | 4 + company2 | 07-02-2023 | 2000 | | | | | | | + company2 | 07-03-2023 | 1500 | | | | | | | + company2 | 07-04-2023 | 1400 | | | | | | | + company2 | 07-05-2023 | 1500 | | | | | | | 0 + company2 | 07-06-2023 | 60 | 60 | 1200 | 1300 | 60 | 3660 | 915.0000000000000000 | 4 + company2 | 07-07-2023 | 1100 | | | | | | | + company2 | 07-08-2023 | 1300 | | | | | | | + company2 | 07-09-2023 | 1200 | | | | | | | + company2 | 07-10-2023 | 1300 | | | | | | | 0 +(20 rows) + +-- using AFTER MATCH SKIP TO NEXT ROW +SELECT company, tdate, price, + first_value(price) OVER w, + last_value(price) OVER w, + max(price) OVER w, + min(price) OVER w, + sum(price) OVER w, + avg(price) OVER w, + count(price) OVER w +FROM stock +WINDOW w AS ( +PARTITION BY company +ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +AFTER MATCH SKIP TO NEXT ROW +INITIAL +PATTERN (START UP+ DOWN+) +DEFINE +START AS TRUE, +UP AS price > PREV(price), +DOWN AS price < PREV(price) +); + company | tdate | price | first_value | last_value | max | min | sum | avg | count +----------+------------+-------+-------------+------------+------+------+------+-----------------------+------- + company1 | 07-01-2023 | 100 | 100 | 140 | 200 | 100 | 590 | 147.5000000000000000 | 4 + company1 | 07-02-2023 | 200 | | | | | | | 0 + company1 | 07-03-2023 | 150 | | | | | | | 0 + company1 | 07-04-2023 | 140 | 140 | 90 | 150 | 90 | 380 | 126.6666666666666667 | 3 + company1 | 07-05-2023 | 150 | | | | | | | 0 + company1 | 07-06-2023 | 90 | 90 | 120 | 130 | 90 | 450 | 112.5000000000000000 | 4 + company1 | 07-07-2023 | 110 | 110 | 120 | 130 | 110 | 360 | 120.0000000000000000 | 3 + company1 | 07-08-2023 | 130 | | | | | | | 0 + company1 | 07-09-2023 | 120 | | | | | | | 0 + company1 | 07-10-2023 | 130 | | | | | | | 0 + company2 | 07-01-2023 | 50 | 50 | 1400 | 2000 | 50 | 4950 | 1237.5000000000000000 | 4 + company2 | 07-02-2023 | 2000 | | | | | | | 0 + company2 | 07-03-2023 | 1500 | | | | | | | 0 + company2 | 07-04-2023 | 1400 | 1400 | 60 | 1500 | 60 | 2960 | 986.6666666666666667 | 3 + company2 | 07-05-2023 | 1500 | | | | | | | 0 + company2 | 07-06-2023 | 60 | 60 | 1200 | 1300 | 60 | 3660 | 915.0000000000000000 | 4 + company2 | 07-07-2023 | 1100 | 1100 | 1200 | 1300 | 1100 | 3600 | 1200.0000000000000000 | 3 + company2 | 07-08-2023 | 1300 | | | | | | | 0 + company2 | 07-09-2023 | 1200 | | | | | | | 0 + company2 | 07-10-2023 | 1300 | | | | | | | 0 +(20 rows) + +-- JOIN case +CREATE TEMP TABLE t1 (i int, v1 int); +CREATE TEMP TABLE t2 (j int, v2 int); +INSERT INTO t1 VALUES(1,10); +INSERT INTO t1 VALUES(1,11); +INSERT INTO t1 VALUES(1,12); +INSERT INTO t2 VALUES(2,10); +INSERT INTO t2 VALUES(2,11); +INSERT INTO t2 VALUES(2,12); +SELECT * FROM t1, t2 WHERE t1.v1 <= 11 AND t2.v2 <= 11; + i | v1 | j | v2 +---+----+---+---- + 1 | 10 | 2 | 10 + 1 | 10 | 2 | 11 + 1 | 11 | 2 | 10 + 1 | 11 | 2 | 11 +(4 rows) + +SELECT *, count(*) OVER w FROM t1, t2 +WINDOW w AS ( + PARTITION BY t1.i + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (A) + DEFINE + A AS v1 <= 11 AND v2 <= 11 +); + i | v1 | j | v2 | count +---+----+---+----+------- + 1 | 10 | 2 | 10 | 1 + 1 | 10 | 2 | 11 | 1 + 1 | 10 | 2 | 12 | 0 + 1 | 11 | 2 | 10 | 1 + 1 | 11 | 2 | 11 | 1 + 1 | 11 | 2 | 12 | 0 + 1 | 12 | 2 | 10 | 0 + 1 | 12 | 2 | 11 | 0 + 1 | 12 | 2 | 12 | 0 +(9 rows) + +-- WITH case +WITH wstock AS ( + SELECT * FROM stock WHERE tdate < '2023-07-08' +) +SELECT tdate, price, +first_value(tdate) OVER w, +count(*) OVER w + FROM wstock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + tdate | price | first_value | count +------------+-------+-------------+------- + 07-01-2023 | 100 | 07-01-2023 | 4 + 07-02-2023 | 200 | | + 07-03-2023 | 150 | | + 07-04-2023 | 140 | | + 07-05-2023 | 150 | | 0 + 07-06-2023 | 90 | | 0 + 07-07-2023 | 110 | | 0 + 07-01-2023 | 50 | 07-01-2023 | 4 + 07-02-2023 | 2000 | | + 07-03-2023 | 1500 | | + 07-04-2023 | 1400 | | + 07-05-2023 | 1500 | | 0 + 07-06-2023 | 60 | | 0 + 07-07-2023 | 1100 | | 0 +(14 rows) + +-- +-- Error cases +-- +-- row pattern definition variable name must not appear more than once +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + ORDER BY tdate + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price), + UP AS price > PREV(price) +); +ERROR: syntax error at or near "ORDER" +LINE 6: ORDER BY tdate + ^ +-- pattern variable name must appear in DEFINE +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+ END) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); +ERROR: syntax error at or near "END" +LINE 8: PATTERN (START UP+ DOWN+ END) + ^ +-- FRAME must start at current row when row patttern recognition is used +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); +ERROR: FRAME must start at current row when row patttern recognition is used +-- SEEK is not supported +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP TO NEXT ROW + SEEK + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); +ERROR: SEEK is not supported +LINE 8: SEEK + ^ +HINT: Use INITIAL. diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index f0987ff537..a6542f3dea 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -98,7 +98,7 @@ test: publication subscription # Another group of parallel tests # select_views depends on create_view # ---------- -test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock indirect_toast equivclass +test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock indirect_toast equivclass rpr # ---------- # Another group of parallel tests (JSON related) diff --git a/src/test/regress/sql/rpr.sql b/src/test/regress/sql/rpr.sql new file mode 100644 index 0000000000..0a69cc0e11 --- /dev/null +++ b/src/test/regress/sql/rpr.sql @@ -0,0 +1,392 @@ +-- +-- Test for row pattern definition clause +-- + +CREATE TEMP TABLE stock ( + company TEXT, + tdate DATE, + price INTEGER +); +INSERT INTO stock VALUES ('company1', '2023-07-01', 100); +INSERT INTO stock VALUES ('company1', '2023-07-02', 200); +INSERT INTO stock VALUES ('company1', '2023-07-03', 150); +INSERT INTO stock VALUES ('company1', '2023-07-04', 140); +INSERT INTO stock VALUES ('company1', '2023-07-05', 150); +INSERT INTO stock VALUES ('company1', '2023-07-06', 90); +INSERT INTO stock VALUES ('company1', '2023-07-07', 110); +INSERT INTO stock VALUES ('company1', '2023-07-08', 130); +INSERT INTO stock VALUES ('company1', '2023-07-09', 120); +INSERT INTO stock VALUES ('company1', '2023-07-10', 130); +INSERT INTO stock VALUES ('company2', '2023-07-01', 50); +INSERT INTO stock VALUES ('company2', '2023-07-02', 2000); +INSERT INTO stock VALUES ('company2', '2023-07-03', 1500); +INSERT INTO stock VALUES ('company2', '2023-07-04', 1400); +INSERT INTO stock VALUES ('company2', '2023-07-05', 1500); +INSERT INTO stock VALUES ('company2', '2023-07-06', 60); +INSERT INTO stock VALUES ('company2', '2023-07-07', 1100); +INSERT INTO stock VALUES ('company2', '2023-07-08', 1300); +INSERT INTO stock VALUES ('company2', '2023-07-09', 1200); +INSERT INTO stock VALUES ('company2', '2023-07-10', 1300); + +SELECT * FROM stock; + +-- basic test using PREV +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- basic test using PREV. UP appears twice +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+ UP+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- basic test using PREV. Use '*' +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP* DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- basic test with none greedy pattern +SELECT company, tdate, price, count(*) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (A A A) + DEFINE + A AS price >= 140 AND price <= 150 +); + +-- last_value() should remain consistent +SELECT company, tdate, price, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- omit "START" in DEFINE but it is ok because "START AS TRUE" is +-- implicitly defined. per spec. +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w, + nth_value(tdate, 2) OVER w AS nth_second + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- the first row start with less than or equal to 100 +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (LOWPRICE UP+ DOWN+) + DEFINE + LOWPRICE AS price <= 100, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- second row raises 120% +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (LOWPRICE UP+ DOWN+) + DEFINE + LOWPRICE AS price <= 100, + UP AS price > PREV(price) * 1.2, + DOWN AS price < PREV(price) +); + +-- using NEXT +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UPDOWN) + DEFINE + START AS TRUE, + UPDOWN AS price > PREV(price) AND price > NEXT(price) +); + +-- using AFTER MATCH SKIP TO NEXT ROW +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP TO NEXT ROW + INITIAL + PATTERN (START UPDOWN) + DEFINE + START AS TRUE, + UPDOWN AS price > PREV(price) AND price > NEXT(price) +); + +-- match everything + +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP PAST LAST ROW + INITIAL + PATTERN (A+) + DEFINE + A AS TRUE +); + +-- backtracking with reclassification of rows +-- using AFTER MATCH SKIP PAST LAST ROW +SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP PAST LAST ROW + INITIAL + PATTERN (A+ B+) + DEFINE + A AS price > 100, + B AS price > 100 +); + +-- backtracking with reclassification of rows +-- using AFTER MATCH SKIP TO NEXT ROW +SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP TO NEXT ROW + INITIAL + PATTERN (A+ B+) + DEFINE + A AS price > 100, + B AS price > 100 +); + +-- ROWS BETWEEN CURRENT ROW AND offset FOLLOWING +SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w, + count(*) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND 2 FOLLOWING + AFTER MATCH SKIP PAST LAST ROW + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- +-- Aggregates +-- + +-- using AFTER MATCH SKIP PAST LAST ROW +SELECT company, tdate, price, + first_value(price) OVER w, + last_value(price) OVER w, + max(price) OVER w, + min(price) OVER w, + sum(price) OVER w, + avg(price) OVER w, + count(price) OVER w +FROM stock +WINDOW w AS ( +PARTITION BY company +ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +AFTER MATCH SKIP PAST LAST ROW +INITIAL +PATTERN (START UP+ DOWN+) +DEFINE +START AS TRUE, +UP AS price > PREV(price), +DOWN AS price < PREV(price) +); + +-- using AFTER MATCH SKIP TO NEXT ROW +SELECT company, tdate, price, + first_value(price) OVER w, + last_value(price) OVER w, + max(price) OVER w, + min(price) OVER w, + sum(price) OVER w, + avg(price) OVER w, + count(price) OVER w +FROM stock +WINDOW w AS ( +PARTITION BY company +ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +AFTER MATCH SKIP TO NEXT ROW +INITIAL +PATTERN (START UP+ DOWN+) +DEFINE +START AS TRUE, +UP AS price > PREV(price), +DOWN AS price < PREV(price) +); + +-- JOIN case +CREATE TEMP TABLE t1 (i int, v1 int); +CREATE TEMP TABLE t2 (j int, v2 int); +INSERT INTO t1 VALUES(1,10); +INSERT INTO t1 VALUES(1,11); +INSERT INTO t1 VALUES(1,12); +INSERT INTO t2 VALUES(2,10); +INSERT INTO t2 VALUES(2,11); +INSERT INTO t2 VALUES(2,12); + +SELECT * FROM t1, t2 WHERE t1.v1 <= 11 AND t2.v2 <= 11; + +SELECT *, count(*) OVER w FROM t1, t2 +WINDOW w AS ( + PARTITION BY t1.i + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (A) + DEFINE + A AS v1 <= 11 AND v2 <= 11 +); + +-- WITH case +WITH wstock AS ( + SELECT * FROM stock WHERE tdate < '2023-07-08' +) +SELECT tdate, price, +first_value(tdate) OVER w, +count(*) OVER w + FROM wstock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- +-- Error cases +-- + +-- row pattern definition variable name must not appear more than once +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + ORDER BY tdate + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price), + UP AS price > PREV(price) +); + +-- pattern variable name must appear in DEFINE +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+ END) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- FRAME must start at current row when row patttern recognition is used +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + INITIAL + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); + +-- SEEK is not supported +SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w + FROM stock + WINDOW w AS ( + PARTITION BY company + ORDER BY tdate + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + AFTER MATCH SKIP TO NEXT ROW + SEEK + PATTERN (START UP+ DOWN+) + DEFINE + START AS TRUE, + UP AS price > PREV(price), + DOWN AS price < PREV(price) +); -- 2.25.1 ----Next_Part(Wed_Nov__8_16_37_05_2023_872)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v11-0007-Allow-to-print-raw-parse-tree.patch" ^ permalink raw reply [nested|flat] 157+ messages in thread
end of thread, other threads:[~2023-11-08 06:57 UTC | newest] Thread overview: 157+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <pryzbyj@telsasoft.com> 2023-11-08 06:57 [PATCH v11 6/7] Row pattern recognition patch (tests). Tatsuo Ishii <ishii@postgresql.org>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox