agora inbox for pgsql-hackers@postgresql.org  

help / color / mirror / Atom feed

[PATCH] Add pg_tablespace_avail() functions
158+ messages / 2 participants
[nested] [flat]

* [PATCH] Add pg_tablespace_avail() functions
@ 2019-11-08 13:12  Christoph Berg <myon@debian.org>
  0 siblings, 0 replies; 158+ messages in thread

From: Christoph Berg @ 2019-11-08 13:12 UTC (permalink / raw)

This exposes the f_avail value from statvfs() on tablespace directories
on the SQL level, allowing monitoring of free disk space from within the
server. On windows, GetDiskFreeSpaceEx() is used.

Permissions required match those from pg_tablespace_size.

In psql, include a new "Free" column in \db+ output.
---
 doc/src/sgml/func.sgml          | 21 ++++++++
 doc/src/sgml/ref/psql-ref.sgml  |  2 +-
 src/backend/utils/adt/dbsize.c  | 94 +++++++++++++++++++++++++++++++++
 src/bin/psql/describe.c         | 11 ++--
 src/include/catalog/pg_proc.dat |  8 +++
 5 files changed, 132 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 51dd8ad6571..0b4456ad958 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -30093,6 +30093,27 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
        </para></entry>
       </row>
 
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        <indexterm>
+         <primary>pg_tablespace_avail</primary>
+        </indexterm>
+        <function>pg_tablespace_avail</function> ( <type>name</type> )
+        <returnvalue>bigint</returnvalue>
+       </para>
+       <para role="func_signature">
+        <function>pg_tablespace_avail</function> ( <type>oid</type> )
+        <returnvalue>bigint</returnvalue>
+       </para>
+       <para>
+        Returns the available disk space in the tablespace with the
+        specified name or OID. To use this function, you must
+        have <literal>CREATE</literal> privilege on the specified tablespace
+        or have privileges of the <literal>pg_read_all_stats</literal> role,
+        unless it is the default tablespace for the current database.
+       </para></entry>
+      </row>
+
       <row>
        <entry role="func_table_entry"><para role="func_signature">
         <indexterm>
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index cedccc14129..9e1bec0b422 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -1492,7 +1492,7 @@ SELECT $1 \parse stmt1
         If <literal>x</literal> is appended to the command name, the results
         are displayed in expanded mode.
         If <literal>+</literal> is appended to the command name, each tablespace
-        is listed with its associated options, on-disk size, permissions and
+        is listed with its associated options, on-disk size and free disk space, permissions and
         description.
         </para>
         </listitem>
diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c
index 25865b660ef..3a2f47c50ec 100644
--- a/src/backend/utils/adt/dbsize.c
+++ b/src/backend/utils/adt/dbsize.c
@@ -12,6 +12,11 @@
 #include "postgres.h"
 
 #include <sys/stat.h>
+#ifdef WIN32
+#include <fileapi.h>
+#else
+#include <sys/statvfs.h>
+#endif
 
 #include "access/htup_details.h"
 #include "access/relation.h"
@@ -316,6 +321,95 @@ pg_tablespace_size_name(PG_FUNCTION_ARGS)
 }
 
 
+/*
+ * Return available disk space of tablespace. Returns -1 if the tablespace
+ * directory cannot be found.
+ */
+static int64
+calculate_tablespace_avail(Oid tblspcOid)
+{
+	char		tblspcPath[MAXPGPATH];
+	AclResult	aclresult;
+#ifdef WIN32
+	ULARGE_INTEGER lpFreeBytesAvailable;
+#else
+	struct statvfs fst;
+#endif
+
+	/*
+	 * User must have privileges of pg_read_all_stats or have CREATE privilege
+	 * for target tablespace, either explicitly granted or implicitly because
+	 * it is default for current database.
+	 */
+	if (tblspcOid != MyDatabaseTableSpace &&
+		!has_privs_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS))
+	{
+		aclresult = object_aclcheck(TableSpaceRelationId, tblspcOid, GetUserId(), ACL_CREATE);
+		if (aclresult != ACLCHECK_OK)
+			aclcheck_error(aclresult, OBJECT_TABLESPACE,
+						   get_tablespace_name(tblspcOid));
+	}
+
+	if (tblspcOid == DEFAULTTABLESPACE_OID)
+		snprintf(tblspcPath, MAXPGPATH, "base");
+	else if (tblspcOid == GLOBALTABLESPACE_OID)
+		snprintf(tblspcPath, MAXPGPATH, "global");
+	else
+		snprintf(tblspcPath, MAXPGPATH, "%s/%u/%s", PG_TBLSPC_DIR, tblspcOid,
+				 TABLESPACE_VERSION_DIRECTORY);
+
+#ifdef WIN32
+	if (GetDiskFreeSpaceEx(tblspcPath, &lpFreeBytesAvailable, NULL, NULL) == false)
+		return -1;
+
+	return lpFreeBytesAvailable.QuadPart; /* ULONGLONG part of ULARGE_INTEGER */
+#else
+	if (statvfs(tblspcPath, &fst) < 0)
+		return -1;
+
+	return fst.f_bavail * fst.f_bsize; /* available blocks times block size */
+#endif
+}
+
+Datum
+pg_tablespace_avail_oid(PG_FUNCTION_ARGS)
+{
+	Oid			tblspcOid = PG_GETARG_OID(0);
+	int64		avail;
+
+	/*
+	 * Not needed for correctness, but avoid non-user-facing error message
+	 * later if the tablespace doesn't exist.
+	 */
+	if (!SearchSysCacheExists1(TABLESPACEOID, ObjectIdGetDatum(tblspcOid)))
+		ereport(ERROR,
+				errcode(ERRCODE_UNDEFINED_OBJECT),
+				errmsg("tablespace with OID %u does not exist", tblspcOid));
+
+	avail = calculate_tablespace_avail(tblspcOid);
+
+	if (avail < 0)
+		PG_RETURN_NULL();
+
+	PG_RETURN_INT64(avail);
+}
+
+Datum
+pg_tablespace_avail_name(PG_FUNCTION_ARGS)
+{
+	Name		tblspcName = PG_GETARG_NAME(0);
+	Oid			tblspcOid = get_tablespace_oid(NameStr(*tblspcName), false);
+	int64		avail;
+
+	avail = calculate_tablespace_avail(tblspcOid);
+
+	if (avail < 0)
+		PG_RETURN_NULL();
+
+	PG_RETURN_INT64(avail);
+}
+
+
 /*
  * calculate size of (one fork of) a relation
  *
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index e6cf468ac9e..8c52a126ac1 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -241,10 +241,15 @@ describeTablespaces(const char *pattern, bool verbose)
 		printACLColumn(&buf, "spcacl");
 		appendPQExpBuffer(&buf,
 						  ",\n  spcoptions AS \"%s\""
-						  ",\n  pg_catalog.pg_size_pretty(pg_catalog.pg_tablespace_size(oid)) AS \"%s\""
-						  ",\n  pg_catalog.shobj_description(oid, 'pg_tablespace') AS \"%s\"",
+						  ",\n  pg_catalog.pg_size_pretty(pg_catalog.pg_tablespace_size(oid)) AS \"%s\"",
 						  gettext_noop("Options"),
-						  gettext_noop("Size"),
+						  gettext_noop("Size"));
+		if (pset.sversion >= 180000)
+			appendPQExpBuffer(&buf,
+							  ",\n  pg_catalog.pg_size_pretty(pg_catalog.pg_tablespace_avail(oid)) AS \"%s\"",
+							  gettext_noop("Free"));
+		appendPQExpBuffer(&buf,
+						  ",\n  pg_catalog.shobj_description(oid, 'pg_tablespace') AS \"%s\"",
 						  gettext_noop("Description"));
 	}
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 42e427f8fe8..9d64da6bfb8 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7680,6 +7680,14 @@
   descr => 'total disk space usage for the specified tablespace',
   proname => 'pg_tablespace_size', provolatile => 'v', prorettype => 'int8',
   proargtypes => 'name', prosrc => 'pg_tablespace_size_name' },
+{ oid => '6015',
+  descr => 'disk stats for the specified tablespace',
+  proname => 'pg_tablespace_avail', provolatile => 'v', prorettype => 'int8',
+  proargtypes => 'oid', prosrc => 'pg_tablespace_avail_oid' },
+{ oid => '6016',
+  descr => 'disk stats for the specified tablespace',
+  proname => 'pg_tablespace_avail', provolatile => 'v', prorettype => 'int8',
+  proargtypes => 'name', prosrc => 'pg_tablespace_avail_name' },
 { oid => '2324', descr => 'total disk space usage for the specified database',
   proname => 'pg_database_size', provolatile => 'v', prorettype => 'int8',
   proargtypes => 'oid', prosrc => 'pg_database_size_oid' },
-- 
2.47.2


--vqKhBUYnSUFh3RIa--





^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH v13 03/18] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index d6567ec231..10d4da136f 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1628,40 +1628,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3085,6 +3078,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip invalid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread

* [PATCH 3/5] WIP: Add SKIPVALID flag for more integration
@ 2020-10-30 21:23  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 158+ messages in thread

From: Justin Pryzby @ 2020-10-30 21:23 UTC (permalink / raw)

XXX: this breaks progress reporting?
---
 src/backend/commands/indexcmds.c | 36 +++++++++++++++-----------------
 src/include/catalog/index.h      |  1 +
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 8f4eab22eb..e54314e9a4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1637,40 +1637,33 @@ DefineIndex(Oid relationId,
 	return address;
 }
 
-/* Reindex invalid child indexes created earlier */
+/*
+ * Reindex invalid child indexes created earlier thereby validating
+ * the parent index.
+ */
 static void
 reindex_invalid_child_indexes(Oid indexRelationId)
 {
-	ListCell *lc;
-	int		npart = 0;
 	ReindexParams params = {
-		.options = REINDEXOPT_CONCURRENTLY
+		.options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_SKIPVALID
 	};
 
-	PreventInTransactionBlock(true, "REINDEX INDEX");
-
-	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
-	{
-		Oid			partoid = lfirst_oid(lc);
-
-		if (!get_index_isvalid(partoid) &&
-				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			ReindexRelationConcurrently(partoid, &params);
-
-		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 npart++);
-	}
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
 	 * it's meaningless for an index without storage).
 	 * This must be done only while holding a lock which precludes adding
 	 * partitions.
-	 * See also: validatePartitionedIndex().
 	 */
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+
+	/*
+	 * Process each partition listed in a separate transaction.  Note that
+	 * this commits and then starts a new transaction immediately.
+	 */
+	ReindexPartitions(indexRelationId, &params, true);
+
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
 }
@@ -3094,6 +3087,11 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel)
 		if (!RELKIND_HAS_STORAGE(partkind))
 			continue;
 
+		/* Skip valid indexes, if requested */
+		if ((params->options & REINDEXOPT_SKIPVALID) != 0 &&
+				get_index_isvalid(partoid))
+			continue;
+
 		Assert(partkind == RELKIND_INDEX ||
 			   partkind == RELKIND_RELATION);
 
diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h
index e22d506436..994fe94fa1 100644
--- a/src/include/catalog/index.h
+++ b/src/include/catalog/index.h
@@ -42,6 +42,7 @@ typedef struct ReindexParams
 #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */
 #define REINDEXOPT_MISSING_OK 	0x04	/* skip missing relations */
 #define REINDEXOPT_CONCURRENTLY	0x08	/* concurrent mode */
+#define REINDEXOPT_SKIPVALID	0x10	/* skip valid indexes */
 
 /* state info for validate_index bulkdelete callback */
 typedef struct ValidateIndexState
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0004-ReindexPartitions-to-set-indisvalid.patch"



^ permalink  raw  reply  [nested|flat] 158+ messages in thread