[PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..

agora inbox for [email protected]  
help / color / mirror / Atom feed

[PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
157+ messages / 2 participants
[nested] [flat]

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once..
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

The utility of this is to reindex N partitions in 2 transactions, rather than
2*N transactions.
---
 src/backend/commands/indexcmds.c           | 258 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 182 insertions(+), 80 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b21555b6cf..f5fea14ff4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -2636,7 +2647,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -3002,20 +3021,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_indexes(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -3060,38 +3128,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
-
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
 
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_indexes(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_indexes(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3254,18 +3342,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3274,13 +3353,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3553,6 +3625,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+		ReindexParams *params, MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -4026,42 +4161,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 830fdddf24..6f41adf736 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v13-0006-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once
@ 2020-11-01 18:25  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Justin Pryzby @ 2020-11-01 18:25 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c           | 262 ++++++++++++++-------
 src/test/regress/expected/create_index.out |   4 +-
 2 files changed, 185 insertions(+), 81 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 739bd14001..08d44a1999 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -102,6 +102,8 @@ static void ReindexMultipleInternal(List *relids,
 									ReindexParams *params);
 static bool ReindexRelationConcurrently(Oid relationOid,
 										ReindexParams *params);
+static List *ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds,
+										int options, MemoryContext private_context);
 static void update_relispartition(Oid relationId, bool newval);
 static inline void set_indexsafe_procflags(void);
 
@@ -124,6 +126,15 @@ typedef struct ReindexErrorInfo
 	char		relkind;
 } ReindexErrorInfo;
 
+/* Argument to ReindexIndexConcurrently takes a List* of these */
+typedef struct ReindexIndexInfo
+{
+	Oid			indexId;
+	Oid			tableId;
+	Oid			amId;
+	bool		safe;		/* for set_indexsafe_procflags */
+} ReindexIndexInfo;
+
 /*
  * CheckIndexCompatible
  *		Determine whether an existing index definition is compatible with a
@@ -1635,7 +1646,9 @@ DefineIndex(Oid relationId,
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY };
+	ReindexParams params = {
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID,
+	};
 
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
@@ -2607,7 +2620,15 @@ ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel)
 		ReindexPartitions(indOid, params, isTopLevel);
 	else if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
 			 persistence != RELPERSISTENCE_TEMP)
-		ReindexRelationConcurrently(indOid, params);
+	{
+		ReindexIndexInfo idxinfo = {
+			.indexId = indOid,
+			/* other fields set later */
+		};
+		ReindexIndexesConcurrently(list_make1(&idxinfo),
+				list_make1_oid(IndexGetRelation(indOid, false)),
+				params->options, CurrentMemoryContext);
+	}
 	else
 	{
 		ReindexParams newparams = *params;
@@ -2938,20 +2959,69 @@ reindex_error_callback(void *arg)
 				   errinfo->relnamespace, errinfo->relname);
 }
 
+
+/*
+ * Given a list of index oids, return a list of leaf partitions by removing
+ * any intermediate parents.  heaprels is populated with the corresponding
+ * tables.
+ */
+static List *
+leaf_partitions(List *inhoids, int options, List **heaprels)
+{
+	List		*partitions = NIL;
+	ListCell	*lc;
+
+	foreach(lc, inhoids)
+	{
+		Oid			partoid = lfirst_oid(lc);
+		Oid			tableoid;
+		Relation	table;
+		char		partkind = get_rel_relkind(partoid);
+
+		/*
+		 * This discards partitioned indexes and foreign tables.
+		 */
+		if (!RELKIND_HAS_STORAGE(partkind))
+			continue;
+
+		Assert(partkind == RELKIND_INDEX);
+
+		/* Skip invalid indexes, if requested */
+		if ((options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
+		/* (try to) Open the table, with lock */
+		tableoid = IndexGetRelation(partoid, false);
+		table = table_open(tableoid, ShareLock);
+		table_close(table, NoLock);
+
+		/* Save partition OID in current MemoryContext */
+		partitions = lappend_oid(partitions, partoid);
+		*heaprels = lappend_oid(*heaprels, tableoid);
+	}
+
+	return partitions;
+}
+
+
 /*
  * ReindexPartitions
  *
  * Reindex a set of partitions, per the partitioned index or table given
  * by the caller.
+ * XXX: should be further refactored with logic from ReindexRelationConcurrently
  */
 static void
 ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 {
-	List	   *partitions = NIL;
+	List	   *partitions = NIL,
+			*heaprels = NIL;
 	char		relkind = get_rel_relkind(relid);
 	char	   *relname = get_rel_name(relid);
 	char	   *relnamespace = get_namespace_name(get_rel_namespace(relid));
 	MemoryContext reindex_context;
+	MemoryContext old_context;
 	List	   *inhoids;
 	ListCell   *lc;
 	ErrorContextCallback errcallback;
@@ -2996,38 +3066,58 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 	 * The list of relations to reindex are the physical partitions of the
 	 * tree so discard any partitioned table or index.
 	 */
-	foreach(lc, inhoids)
-	{
-		Oid			partoid = lfirst_oid(lc);
-		char		partkind = get_rel_relkind(partoid);
-		MemoryContext old_context;
 
-		/*
-		 * This discards partitioned tables, partitioned indexes and foreign
-		 * tables.
-		 */
-		if (!RELKIND_HAS_STORAGE(partkind))
-			continue;
-
-		/* Skip invalid indexes, if requested */
-		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
-				get_index_isvalid(partoid))
-			continue;
+	if (relkind == RELKIND_PARTITIONED_INDEX)
+	{
+		old_context = MemoryContextSwitchTo(reindex_context);
+		partitions = leaf_partitions(inhoids, params->options, &heaprels);
+		MemoryContextSwitchTo(old_context);
+	} else {
+		/* Loop over parent tables */
+		foreach(lc, inhoids)
+		{
+			Oid		partoid = lfirst_oid(lc);
+			Relation parttable;
+			List	*partindexes;
+
+			parttable = table_open(partoid, ShareLock);
+			old_context = MemoryContextSwitchTo(reindex_context);
+			partindexes = RelationGetIndexList(parttable);
+			partindexes = leaf_partitions(partindexes, params->options, &heaprels);
+			partitions = list_concat(partitions, partindexes);
+
+			MemoryContextSwitchTo(old_context);
+			table_close(parttable, ShareLock);
+		}
+	}
 
-		Assert(partkind == RELKIND_INDEX ||
-			   partkind == RELKIND_RELATION);
+	if ((params->options & REINDEXOPT_CONCURRENTLY) != 0 &&
+		relkind == RELKIND_PARTITIONED_INDEX &&
+		get_rel_persistence(relid) != RELPERSISTENCE_TEMP)
+	{
+		List			   *idxinfos = NIL;
+		ReindexIndexInfo	*idxinfo;
 
-		/* Save partition OID */
 		old_context = MemoryContextSwitchTo(reindex_context);
-		partitions = lappend_oid(partitions, partoid);
+		foreach (lc, partitions)
+		{
+			Oid partoid = lfirst_oid(lc);
+			idxinfo = palloc(sizeof(ReindexIndexInfo));
+			idxinfo->indexId = partoid;
+			/* other fields set later */
+			idxinfos = lappend(idxinfos, idxinfo);
+		}
 		MemoryContextSwitchTo(old_context);
-	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 */
-	ReindexMultipleInternal(partitions, params);
+		/* Process all indexes in a single loop */
+		ReindexIndexesConcurrently(idxinfos, heaprels, params->options, reindex_context);
+	} else {
+		/*
+		 * Process each partition listed in a separate transaction.  Note that
+		 * this commits and then starts a new transaction immediately.
+		 */
+		ReindexMultipleInternal(partitions, params);
+	}
 
 	/*
 	 * If indexes exist on all of the partitioned table's children, and we
@@ -3172,18 +3262,9 @@ ReindexMultipleInternal(List *relids, ReindexParams *params)
 static bool
 ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 {
-	typedef struct ReindexIndexInfo
-	{
-		Oid			indexId;
-		Oid			tableId;
-		Oid			amId;
-		bool		safe;		/* for set_indexsafe_procflags */
-	} ReindexIndexInfo;
 	List	   *heapRelationIds = NIL;
 	List	   *indexIds = NIL;
 	List	   *newIndexIds = NIL;
-	List	   *relationLocks = NIL;
-	List	   *lockTags = NIL;
 	ListCell   *lc,
 			   *lc2;
 	MemoryContext private_context;
@@ -3192,13 +3273,6 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	char	   *relationName = NULL;
 	char	   *relationNamespace = NULL;
 	PGRUsage	ru0;
-	const int	progress_index[] = {
-		PROGRESS_CREATEIDX_COMMAND,
-		PROGRESS_CREATEIDX_PHASE,
-		PROGRESS_CREATEIDX_INDEX_OID,
-		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
-	};
-	int64		progress_vals[4];
 
 	/*
 	 * Create a memory context that will survive forced transaction commits we
@@ -3449,6 +3523,69 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 
 	Assert(heapRelationIds != NIL);
 
+	/* Do the work */
+	newIndexIds = ReindexIndexesConcurrently(indexIds, heapRelationIds, params->options, private_context);
+
+	/* Log what we did */
+	if ((params->options & REINDEXOPT_VERBOSE) != 0)
+	{
+		if (relkind == RELKIND_INDEX)
+			ereport(INFO,
+					(errmsg("index \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		else
+		{
+			foreach(lc, newIndexIds)
+			{
+				Oid			indOid = lfirst_oid(lc);
+				ereport(INFO,
+						(errmsg("index \"%s.%s\" was reindexed",
+								get_namespace_name(get_rel_namespace(indOid)),
+								get_rel_name(indOid))));
+				/* Don't show rusage here, since it's not per index. */
+			}
+
+			ereport(INFO,
+					(errmsg("table \"%s.%s\" was reindexed",
+							relationNamespace, relationName),
+					 errdetail("%s.",
+							   pg_rusage_show(&ru0))));
+		}
+	}
+
+
+	MemoryContextDelete(private_context);
+
+	return true;
+}
+
+/*
+ * Reindex concurrently for an arbitrary list of index relations
+ * This is called by ReindexRelationConcurrently and
+ */
+static List *
+ReindexIndexesConcurrently(List *indexIds, List *heapRelationIds, int options,
+		MemoryContext private_context)
+{
+	List	   *newIndexIds = NIL;
+	List	   *relationLocks = NIL;
+	List	   *lockTags = NIL;
+
+	ListCell   *lc,
+			   *lc2;
+
+	MemoryContext oldcontext;
+
+	const int	progress_index[] = {
+		PROGRESS_CREATEIDX_COMMAND,
+		PROGRESS_CREATEIDX_PHASE,
+		PROGRESS_CREATEIDX_INDEX_OID,
+		PROGRESS_CREATEIDX_ACCESS_METHOD_OID
+	};
+	int64		progress_vals[4];
+
 	/*-----
 	 * Now we have all the indexes we want to process in indexIds.
 	 *
@@ -3913,42 +4050,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params)
 	/* Start a new transaction to finish process properly */
 	StartTransactionCommand();
 
-	/* Log what we did */
-	if ((params->options & REINDEXOPT_VERBOSE) != 0)
-	{
-		if (relkind == RELKIND_INDEX)
-			ereport(INFO,
-					(errmsg("index \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		else
-		{
-			foreach(lc, newIndexIds)
-			{
-				ReindexIndexInfo *idx = lfirst(lc);
-				Oid			indOid = idx->indexId;
-
-				ereport(INFO,
-						(errmsg("index \"%s.%s\" was reindexed",
-								get_namespace_name(get_rel_namespace(indOid)),
-								get_rel_name(indOid))));
-				/* Don't show rusage here, since it's not per index. */
-			}
-
-			ereport(INFO,
-					(errmsg("table \"%s.%s\" was reindexed",
-							relationNamespace, relationName),
-					 errdetail("%s.",
-							   pg_rusage_show(&ru0))));
-		}
-	}
-
-	MemoryContextDelete(private_context);
-
 	pgstat_progress_end_command();
 
-	return true;
+	return newIndexIds;
 }
 
 /*
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index fc6afab58a..4a03ab2abb 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2470,12 +2470,12 @@ COMMIT;
 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 -- These are the toast table and index of pg_authid.
 REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table
 ERROR:  cannot reindex system catalogs concurrently
 REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index
-ERROR:  cannot reindex system catalogs concurrently
+ERROR:  concurrent index creation on system catalog tables is not supported
 REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM
 ERROR:  cannot reindex system catalogs concurrently
 -- Warns about catalog relations
-- 
2.17.0


--MfFXiAuoTsnnDAfZ
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v12-0005-More-refactoring.patch"



^ permalink  raw  reply  [nested|flat] 157+ messages in thread

* [PATCH v4 2/2] Protect role resolution in roleSpecsToIds() against concurrent DROP
@ 2026-07-06 08:28  Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 157+ messages in thread

From: Bertrand Drouvot @ 2026-07-06 08:28 UTC (permalink / raw)

roleSpecsToIds() resolves role names to OIDs without acquiring any lock.
A concurrent DROP ROLE that commits between this resolution and the caller's use
of the OID leaves the caller operating on a stale OID, which can create orphaned
pg_auth_members entries.

Fix this by acquiring AccessShareLock on each resolved role within
roleSpecsToIds(), ensuring the role cannot be dropped while any caller is using
its OID.

The AccessShareLock is held until end of transaction, so an open transaction
that performed GRANT, CREATE ROLE ... ROLE, or REASSIGN OWNED BY will block a
concurrent DROP ROLE on the same role until it commits.

Note that this introduces a potential deadlock between GRANT and DROP ROLE
when both target overlapping roles. The deadlock is detected and one session is
aborted with an error, which is preferable to the pre-patch behavior of silently
creating orphaned catalog entries.

Author: Bertrand Drouvot <[email protected]>
Reported-by: Virender Singla <[email protected]>
Reviewed-by: Surya Poondla <[email protected]>
Discussion: https://postgr.es/m/aki6fMNLUx6%2BBR8K%40bdtpg
Discussion: https://postgr.es/m/CAM6Zo8woa62ZFHtMKox6a4jb8qQ%3Dw87R2L0K8347iE-juQL2EA%40mail.gmail.com
---
 src/backend/commands/user.c                   | 20 +++-
 .../expected/role-membership-drop-member.out  | 97 +++++++++++++++++++
 src/test/isolation/isolation_schedule         |  1 +
 .../specs/role-membership-drop-member.spec    | 63 ++++++++++++
 4 files changed, 180 insertions(+), 1 deletion(-)
  11.5% src/backend/commands/
  49.2% src/test/isolation/expected/
  38.6% src/test/isolation/specs/

diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c
index 5b869e91c17..ab3cfb0b13f 100644
--- a/src/backend/commands/user.c
+++ b/src/backend/commands/user.c
@@ -1778,6 +1778,8 @@ ReassignOwnedObjects(ReassignOwnedStmt *stmt)
  * roleSpecsToIds
  *
  * Given a list of RoleSpecs, generate a list of role OIDs in the same order.
+ * Each role is locked with AccessShareLock to prevent concurrent DROP ROLE
+ * from removing it between resolution and the caller's catalog update.
  *
  * ROLESPEC_PUBLIC is not allowed.
  */
@@ -1792,7 +1794,23 @@ roleSpecsToIds(List *memberNames)
 		RoleSpec   *rolespec = lfirst_node(RoleSpec, l);
 		Oid			roleid;
 
-		roleid = get_rolespec_oid(rolespec, false);
+		if (rolespec->roletype == ROLESPEC_CSTRING)
+			roleid = RoleNameGetOid(rolespec->rolename,
+									AccessShareLock, false,
+									NULL, NULL);
+		else
+		{
+			roleid = get_rolespec_oid(rolespec, false);
+			LockSharedObject(AuthIdRelationId, roleid, 0,
+							 AccessShareLock);
+
+			/* Recheck that the role still exists after locking. */
+			if (!SearchSysCacheExists1(AUTHOID, ObjectIdGetDatum(roleid)))
+				ereport(ERROR,
+						(errcode(ERRCODE_UNDEFINED_OBJECT),
+						 errmsg("role with OID %u does not exist", roleid)));
+		}
+
 		result = lappend_oid(result, roleid);
 	}
 	return result;
diff --git a/src/test/isolation/expected/role-membership-drop-member.out b/src/test/isolation/expected/role-membership-drop-member.out
new file mode 100644
index 00000000000..7daf48395ca
--- /dev/null
+++ b/src/test/isolation/expected/role-membership-drop-member.out
@@ -0,0 +1,97 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1_begin s1_grant s2_drop_member s1_commit s2_check_orphans
+step s1_begin: BEGIN;
+step s1_grant: GRANT regress_role_group TO regress_role_member;
+step s2_drop_member: DROP ROLE regress_role_member; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_member: <... completed>
+step s2_check_orphans: 
+	SELECT count(*)
+	FROM pg_auth_members m
+	LEFT JOIN pg_authid ra ON m.roleid  = ra.oid
+	LEFT JOIN pg_authid me ON m.member  = me.oid
+	LEFT JOIN pg_authid gr ON m.grantor = gr.oid
+	WHERE ra.oid IS NULL OR me.oid IS NULL OR gr.oid IS NULL;
+
+count
+-----
+    0
+(1 row)
+
+
+starting permutation: s1_begin s1_alter_add s2_drop_member s1_commit s2_check_orphans
+step s1_begin: BEGIN;
+step s1_alter_add: ALTER GROUP regress_role_group ADD USER regress_role_member;
+step s2_drop_member: DROP ROLE regress_role_member; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_member: <... completed>
+step s2_check_orphans: 
+	SELECT count(*)
+	FROM pg_auth_members m
+	LEFT JOIN pg_authid ra ON m.roleid  = ra.oid
+	LEFT JOIN pg_authid me ON m.member  = me.oid
+	LEFT JOIN pg_authid gr ON m.grantor = gr.oid
+	WHERE ra.oid IS NULL OR me.oid IS NULL OR gr.oid IS NULL;
+
+count
+-----
+    0
+(1 row)
+
+
+starting permutation: s1_begin s1_create_role s2_drop_member s1_commit s2_check_orphans
+step s1_begin: BEGIN;
+step s1_create_role: CREATE ROLE regress_role_new ROLE regress_role_member;
+step s2_drop_member: DROP ROLE regress_role_member; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_member: <... completed>
+step s2_check_orphans: 
+	SELECT count(*)
+	FROM pg_auth_members m
+	LEFT JOIN pg_authid ra ON m.roleid  = ra.oid
+	LEFT JOIN pg_authid me ON m.member  = me.oid
+	LEFT JOIN pg_authid gr ON m.grantor = gr.oid
+	WHERE ra.oid IS NULL OR me.oid IS NULL OR gr.oid IS NULL;
+
+count
+-----
+    0
+(1 row)
+
+
+starting permutation: s1_begin s1_drop_owned s2_drop_member s1_commit
+step s1_begin: BEGIN;
+step s1_drop_owned: DROP OWNED BY regress_role_member;
+step s2_drop_member: DROP ROLE regress_role_member; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_member: <... completed>
+
+starting permutation: s1_begin s1_reassign_owned s2_drop_member s1_commit
+step s1_begin: BEGIN;
+step s1_reassign_owned: REASSIGN OWNED BY regress_role_member TO regress_role_group;
+step s2_drop_member: DROP ROLE regress_role_member; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_member: <... completed>
+
+starting permutation: s1_begin s1_set_role s1_grant_cu s2_drop_member_cu s1_commit s1_reset s2_check_orphans
+step s1_begin: BEGIN;
+step s1_set_role: SET ROLE regress_role_member_cu;
+step s1_grant_cu: GRANT regress_role_group_cu TO CURRENT_USER;
+step s2_drop_member_cu: DROP ROLE regress_role_member_cu; <waiting ...>
+step s1_commit: COMMIT;
+step s2_drop_member_cu: <... completed>
+step s1_reset: RESET ROLE;
+step s2_check_orphans: 
+	SELECT count(*)
+	FROM pg_auth_members m
+	LEFT JOIN pg_authid ra ON m.roleid  = ra.oid
+	LEFT JOIN pg_authid me ON m.member  = me.oid
+	LEFT JOIN pg_authid gr ON m.grantor = gr.oid
+	WHERE ra.oid IS NULL OR me.oid IS NULL OR gr.oid IS NULL;
+
+count
+-----
+    0
+(1 row)
+
diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule
index b8ebe92553c..8fb8b52b77f 100644
--- a/src/test/isolation/isolation_schedule
+++ b/src/test/isolation/isolation_schedule
@@ -128,3 +128,4 @@ test: matview-write-skew
 test: lock-nowait
 test: for-portion-of
 test: ddl-dependency-locking
+test: role-membership-drop-member
diff --git a/src/test/isolation/specs/role-membership-drop-member.spec b/src/test/isolation/specs/role-membership-drop-member.spec
new file mode 100644
index 00000000000..054326ac535
--- /dev/null
+++ b/src/test/isolation/specs/role-membership-drop-member.spec
@@ -0,0 +1,63 @@
+# Test that role membership commands properly lock the grantee/member
+# role to prevent concurrent DROP ROLE from creating orphaned pg_auth_members
+# entries, or from operating on a stale OID.
+
+setup
+{
+	CREATE ROLE regress_role_group;
+	CREATE ROLE regress_role_member;
+	CREATE ROLE regress_role_group_cu;
+	CREATE ROLE regress_role_member_cu;
+	GRANT regress_role_group_cu TO regress_role_member_cu WITH ADMIN OPTION;
+}
+
+teardown
+{
+	DROP ROLE IF EXISTS regress_role_group;
+	DROP ROLE IF EXISTS regress_role_member;
+	DROP ROLE IF EXISTS regress_role_new;
+	DROP ROLE IF EXISTS regress_role_member_cu;
+	DROP ROLE IF EXISTS regress_role_group_cu;
+}
+
+session s1
+step s1_begin		{ BEGIN; }
+step s1_grant		{ GRANT regress_role_group TO regress_role_member; }
+step s1_alter_add	{ ALTER GROUP regress_role_group ADD USER regress_role_member; }
+step s1_create_role	{ CREATE ROLE regress_role_new ROLE regress_role_member; }
+step s1_drop_owned	{ DROP OWNED BY regress_role_member; }
+step s1_reassign_owned	{ REASSIGN OWNED BY regress_role_member TO regress_role_group; }
+step s1_set_role	{ SET ROLE regress_role_member_cu; }
+step s1_grant_cu	{ GRANT regress_role_group_cu TO CURRENT_USER; }
+step s1_commit		{ COMMIT; }
+step s1_reset		{ RESET ROLE; }
+
+session s2
+step s2_drop_member	{ DROP ROLE regress_role_member; }
+step s2_drop_member_cu	{ DROP ROLE regress_role_member_cu; }
+step s2_check_orphans	{
+	SELECT count(*)
+	FROM pg_auth_members m
+	LEFT JOIN pg_authid ra ON m.roleid  = ra.oid
+	LEFT JOIN pg_authid me ON m.member  = me.oid
+	LEFT JOIN pg_authid gr ON m.grantor = gr.oid
+	WHERE ra.oid IS NULL OR me.oid IS NULL OR gr.oid IS NULL;
+}
+
+# GRANT role TO member and concurrent DROP of the member
+permutation s1_begin s1_grant s2_drop_member s1_commit s2_check_orphans
+
+# ALTER GROUP ADD USER and concurrent DROP of the member
+permutation s1_begin s1_alter_add s2_drop_member s1_commit s2_check_orphans
+
+# CREATE ROLE ... ROLE member and concurrent DROP of the member
+permutation s1_begin s1_create_role s2_drop_member s1_commit s2_check_orphans
+
+# DROP OWNED BY role and concurrent DROP of the role
+permutation s1_begin s1_drop_owned s2_drop_member s1_commit
+
+# REASSIGN OWNED BY role and concurrent DROP of the role
+permutation s1_begin s1_reassign_owned s2_drop_member s1_commit
+
+# GRANT role TO CURRENT_USER and concurrent DROP of the current user
+permutation s1_begin s1_set_role s1_grant_cu s2_drop_member_cu s1_commit s1_reset s2_check_orphans
-- 
2.34.1


--fotX2lJRknAHpfH+--






^ permalink  raw  reply  [nested|flat] 157+ messages in thread

end of thread, other threads:[~2026-07-06 08:28 UTC | newest]

Thread overview: 157+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v13 05/18] Refactor to allow reindexing all index partitions at once.. Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2020-11-01 18:25 [PATCH v12 4/5] Refactor to allow reindexing all index partitions at once Justin Pryzby <[email protected]>
2026-07-06 08:28 [PATCH v4 2/2] Protect role resolution in roleSpecsToIds() against concurrent DROP Bertrand Drouvot <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox