agora inbox for pgsql-hackers@postgresql.org  

help / color / mirror / Atom feed

[PATCH 3/5] WIP: Add SKIPVALID flag for more integration
158+ messages / 2 participants
[nested] [flat]

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread