agora inbox for pgsql-hackers@postgresql.org  

help / color / mirror / Atom feed

[PATCH v38 03/11] Allow to prolong life span of transition tables until transaction end
158+ messages / 2 participants
[nested] [flat]

* [PATCH v38 03/11] Allow to prolong life span of transition tables until transaction end
@ 2019-12-20 01:09 Yugo Nagata <nagata@sraoss.co.jp>
  0 siblings, 0 replies; 158+ messages in thread

From: Yugo Nagata @ 2019-12-20 01:09 UTC (permalink / raw)

Originally, tuplestores of AFTER trigger's transition tables were
freed for each query depth. For our IVM implementation, we would like
to prolong life of the tuplestores because we have to preserve them
for a whole query assuming that some base tables might be changed
in some trigger functions.
---
 src/backend/commands/trigger.c | 80 ++++++++++++++++++++++++++++++++--
 src/include/commands/trigger.h |  2 +
 2 files changed, 78 insertions(+), 4 deletions(-)

diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index b87b4b40d07..49fe531198a 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -3819,6 +3819,10 @@ typedef struct AfterTriggerEventList
  * end of the list, so it is relatively easy to discard them.  The event
  * list chunks themselves are stored in event_cxt.
  *
+ * prolonged_tuplestored is a list of transition table tuplestores whose
+ * life are prolonged to the end of the outmost query instead of each nested
+ * query.
+ *
  * query_depth is the current depth of nested AfterTriggerBeginQuery calls
  * (-1 when the stack is empty).
  *
@@ -3884,6 +3888,7 @@ typedef struct AfterTriggersData
 	SetConstraintState state;	/* the active S C state */
 	AfterTriggerEventList events;	/* deferred-event list */
 	MemoryContext event_cxt;	/* memory context for events, if any */
+	List   *prolonged_tuplestores;	/* list of prolonged tuplestores */
 
 	/* per-query-level data: */
 	AfterTriggersQueryData *query_stack;	/* array of structs shown below */
@@ -3932,6 +3937,7 @@ struct AfterTriggersTableData
 	bool		closed;			/* true when no longer OK to add tuples */
 	bool		before_trig_done;	/* did we already queue BS triggers? */
 	bool		after_trig_done;	/* did we already queue AS triggers? */
+	bool		prolonged;			/* are transition tables prolonged? */
 	AfterTriggerEventList after_trig_events;	/* if so, saved list pointer */
 
 	/* "old" transition table for UPDATE/DELETE, if any */
@@ -3978,6 +3984,7 @@ static void TransitionTableAddTuple(EState *estate,
 									TupleTableSlot *original_insert_tuple,
 									Tuplestorestate *tuplestore);
 static void AfterTriggerFreeQuery(AfterTriggersQueryData *qs);
+static void release_or_prolong_tuplestore(Tuplestorestate *ts, bool prolonged);
 static SetConstraintState SetConstraintStateCreate(int numalloc);
 static SetConstraintState SetConstraintStateCopy(SetConstraintState origstate);
 static SetConstraintState SetConstraintStateAddItem(SetConstraintState state,
@@ -4873,6 +4880,45 @@ afterTriggerInvokeEvents(AfterTriggerEventList *events,
 }
 
 
+/*
+ * SetTransitionTablePreserved
+ *
+ * Prolong lifespan of transition tables corresponding specified relid and
+ * command type to the end of the outmost query instead of each nested query.
+ * This enables to use nested AFTER trigger's transition tables from outer
+ * query's triggers.  Currently, only immediate incremental view maintenance
+ * uses this.
+ */
+void
+SetTransitionTablePreserved(Oid relid, CmdType cmdType)
+{
+	AfterTriggersTableData *table;
+	AfterTriggersQueryData *qs;
+	bool		found = false;
+	ListCell   *lc;
+
+	/* Check state, like AfterTriggerSaveEvent. */
+	if (afterTriggers.query_depth < 0)
+		elog(ERROR, "SetTransitionTablePreserved() called outside of query");
+
+	qs = &afterTriggers.query_stack[afterTriggers.query_depth];
+
+	foreach(lc, qs->tables)
+	{
+		table = (AfterTriggersTableData *) lfirst(lc);
+		if (table->relid == relid && table->cmdType == cmdType &&
+			table->closed)
+		{
+			table->prolonged = true;
+			found = true;
+		}
+	}
+
+	if (!found)
+		elog(ERROR,"could not find table with OID %d and command type %d", relid, cmdType);
+}
+
+
 /*
  * GetAfterTriggersTableData
  *
@@ -5113,6 +5159,7 @@ AfterTriggerBeginXact(void)
 	afterTriggers.firing_depth = 0;
 	afterTriggers.batch_callbacks = NIL;
 	afterTriggers.firing_batch_callbacks = false;
+	afterTriggers.prolonged_tuplestores = NIL;
 
 	/*
 	 * Verify that there is no leftover state remaining.  If these assertions
@@ -5287,11 +5334,11 @@ AfterTriggerFreeQuery(AfterTriggersQueryData *qs)
 		ts = table->old_tuplestore;
 		table->old_tuplestore = NULL;
 		if (ts)
-			tuplestore_end(ts);
+			release_or_prolong_tuplestore(ts, table->prolonged);
 		ts = table->new_tuplestore;
 		table->new_tuplestore = NULL;
 		if (ts)
-			tuplestore_end(ts);
+			release_or_prolong_tuplestore(ts, table->prolonged);
 		if (table->storeslot)
 		{
 			TupleTableSlot *slot = table->storeslot;
@@ -5309,8 +5356,33 @@ AfterTriggerFreeQuery(AfterTriggersQueryData *qs)
 	qs->tables = NIL;
 	list_free_deep(tables);
 
-	list_free_deep(qs->batch_callbacks);
-	qs->batch_callbacks = NIL;
+	/* Release prolonged tuplestores at the end of the outmost query */
+	if (afterTriggers.query_depth == 0)
+	{
+		foreach(lc, afterTriggers.prolonged_tuplestores)
+		{
+			ts = (Tuplestorestate *) lfirst(lc);
+			if (ts)
+				tuplestore_end(ts);
+		}
+		afterTriggers.prolonged_tuplestores = NIL;
+	}
+}
+
+/*
+ * Release the tuplestore, or append it to the prolonged tuplestores list.
+ */
+static void
+release_or_prolong_tuplestore(Tuplestorestate *ts, bool prolonged)
+{
+	if (prolonged && afterTriggers.query_depth > 0)
+	{
+		MemoryContext oldcxt = MemoryContextSwitchTo(CurTransactionContext);
+		afterTriggers.prolonged_tuplestores = lappend(afterTriggers.prolonged_tuplestores, ts);
+		MemoryContextSwitchTo(oldcxt);
+	}
+	else
+		tuplestore_end(ts);
 }
 
 
diff --git a/src/include/commands/trigger.h b/src/include/commands/trigger.h
index 0c3d485abf4..f0f052cc759 100644
--- a/src/include/commands/trigger.h
+++ b/src/include/commands/trigger.h
@@ -269,6 +269,8 @@ extern void AfterTriggerEndSubXact(bool isCommit);
 extern void AfterTriggerSetState(ConstraintsSetStmt *stmt);
 extern bool AfterTriggerPendingOnRel(Oid relid);
 
+extern void SetTransitionTablePreserved(Oid relid, CmdType cmdType);
+
 
 /*
  * in utils/adt/ri_triggers.c
-- 
2.43.0


--Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ
Content-Type: text/x-diff;
 name="v38-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
 filename="v38-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23 Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread