Subject: [PATCH v8 3/8] f! progress reporting..

From: Justin Pryzby <[email-masked]>
Date: Mon, 8 Feb 2021 20:45:26 -0600

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);

 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0

--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"