public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v8 3/8] f! progress reporting..
73+ messages / 4 participants
[nested] [flat]

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v10 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 56018745c8..28493c7931 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6293,6 +6293,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5f2541d316..0256809f06 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1142,7 +1142,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 662aff04b4..1d8a7ca774 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index c6b139d57d..d85781ae0b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9b59a7b4a5..a8bcc599ac 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--mvpLiMfbWzRoNl4x
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v10-0004-Propagate-changes-to-indisclustered-to-child-par.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 64 +++++++++++++++++-----------
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 4 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+					PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
 			/* Do the job. */
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report vacuum full or cluster and number of partitions */
+			progress_val[0] = OidIsValid(indexOid) ?
+				PROGRESS_CLUSTER_COMMAND_CLUSTER :
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+			progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	ReindexParams reindex_params = {0};
 	int			i;
 
-	/* Report that we are now swapping relation files */
+	/* Report that we are now swapping relation files XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
 
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
 	else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
 		reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
 
-	/* Report that we are now reindexing relations */
+	/* Report that we are now reindexing relations XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
 
 	reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
 
-	/* Report that we are now doing clean up */
+	/* Report that we are now doing clean up XXX */
 	pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
 								 PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
 
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					childnum++);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)

This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.

This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
 doc/src/sgml/monitoring.sgml         | 21 +++++++++++
 src/backend/catalog/system_views.sql |  4 +-
 src/backend/commands/cluster.c       | 55 ++++++++++++++++------------
 src/backend/commands/vacuum.c        |  5 +++
 src/include/commands/progress.h      |  3 ++
 src/test/regress/expected/rules.out  |  4 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
        is <literal>rebuilding index</literal>.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>total_partitions</structfield> <type>bigint</type>
+      </para>
+       When clustering a partitioned table, this column is set to the total
+       number of partitions to be clustered.
+      <para>
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>done_partitions</structfield> <type>bigint</type>
+      </para>
+      <para>
+       When clustering a partitioned table, this column is set to the number
+       of partitions which have been clustered.
+      </para></entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
         S.param5 AS heap_tuples_written,
         S.param6 AS heap_blks_total,
         S.param7 AS heap_blks_scanned,
-        S.param8 AS index_rebuild_count
+        S.param8 AS index_rebuild_count,
+        S.param18 AS partitions_total,
+        S.param19 AS partitions_done
     FROM pg_stat_get_progress_info('CLUSTER') AS S
         LEFT JOIN pg_database D ON S.datid = D.oid;
 
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
 static List *get_tables_to_cluster(MemoryContext cluster_context);
 static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
 		Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
 
 
 /*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		/* close relation, keep lock till commit */
 		table_close(rel, ShareUpdateExclusiveLock);
 
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
 		if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			/* Do the job. */
+			pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+					PROGRESS_CLUSTER_COMMAND_CLUSTER);
 			cluster_rel(tableOid, indexOid, &params);
+			pgstat_progress_end_command();
 		}
 		else
 		{
 			List	   *rvs;
 			MemoryContext cluster_context;
 
+			int64		progress_val[2];
+			const int	progress_index[2] = {
+				PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+			};
+
 			/* Refuse to hold strong locks in a user transaction */
 			PreventInTransactionBlock(isTopLevel, "CLUSTER");
 
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 												ALLOCSET_DEFAULT_SIZES);
 
 			rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
-			cluster_multiple_rels(rvs, params.options);
+
+			/* Report command and number of partitions */
+			progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+			/* XXX: rvs includes the parent, which is not a "partition" */
+			progress_val[1] = list_length(rvs);
+
+			pgstat_progress_update_multi_param(2, progress_index, progress_val);
+			cluster_multiple_rels(rvs, params.options, true);
+			pgstat_progress_end_command();
 
 			/* Start a new transaction for the cleanup work. */
 			StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
 		 * cluster_context.
 		 */
 		rvs = get_tables_to_cluster(cluster_context);
-		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_CLUSTER);
+		cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+		pgstat_progress_end_command();
 
 		/* Start a new transaction for the cleanup work. */
 		StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	/* Check for user-requested abort. */
 	CHECK_FOR_INTERRUPTS();
 
-	pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
-	if (OidIsValid(indexOid))
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_CLUSTER);
-	else
-		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
-									 PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
 	/*
 	 * We grab exclusive access to the target rel and index for the duration
 	 * of the transaction.  (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 
 	/* If the table has gone away, we can skip processing it */
 	if (!OldHeap)
-	{
-		pgstat_progress_end_command();
 		return;
-	}
 
 	/*
 	 * Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (!pg_class_ownercheck(tableOid, GetUserId()))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		if (RELATION_IS_OTHER_TEMP(OldHeap))
 		{
 			relation_close(OldHeap, AccessExclusiveLock);
-			pgstat_progress_end_command();
 			return;
 		}
 
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 			if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 					!get_index_isclustered(indexOid))
 			{
 				relation_close(OldHeap, AccessExclusiveLock);
-				pgstat_progress_end_command();
 				return;
 			}
 		}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 		!RelationIsPopulated(OldHeap))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
 	{
 		relation_close(OldHeap, AccessExclusiveLock);
-		pgstat_progress_end_command();
 		return;
 	}
 
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
 	rebuild_relation(OldHeap, indexOid, verbose);
 
 	/* NB: rebuild_relation does table_close() on OldHeap */
-
-	pgstat_progress_end_command();
 }
 
 /*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
 
 /* Cluster each relation in a separate transaction */
 static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
 {
 	ListCell *lc;
+	off_t	  childnum = 0;
 
 	/* Commit to get out of starting transaction */
 	PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
 		cluster_rel(rvtc->tableOid, rvtc->indexOid,
 					&cluster_params);
 
+		if (ispartitioned)
+			pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+					++childnum);
+
+
 		PopActiveSnapshot();
 		CommitTransactionCommand();
 	}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
 #include "catalog/pg_namespace.h"
 #include "commands/cluster.h"
 #include "commands/defrem.h"
+#include "commands/progress.h"
 #include "commands/vacuum.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
 			cluster_params.options |= CLUOPT_VERBOSE;
 
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+		pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+		pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+				PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
 		cluster_rel(relid, InvalidOid, &cluster_params);
+		pgstat_progress_end_command();
 	}
 	else
 		table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
 #define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS		5
 #define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED		6
 #define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT	7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL	17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE	18
 
 /* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
 #define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP	1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
     s.param5 AS heap_tuples_written,
     s.param6 AS heap_blks_total,
     s.param7 AS heap_blks_scanned,
-    s.param8 AS index_rebuild_count
+    s.param8 AS index_rebuild_count,
+    s.param18 AS partitions_total,
+    s.param19 AS partitions_done
    FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
      LEFT JOIN pg_database d ON ((s.datid = d.oid)));
 pg_stat_progress_copy| SELECT s.pid,
-- 
2.17.0


--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* [PATCH 2/5] f! progress reporting
@ 2021-02-15 00:31  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 73+ messages in thread

From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)

---
 src/backend/commands/indexcmds.c | 33 +++++++-------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 9ab1a66971..8f4eab22eb 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1647,40 +1647,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 		.options = REINDEXOPT_CONCURRENTLY
 	};
 
-	MemoryContext	ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
-			ALLOCSET_DEFAULT_SIZES);
-	MemoryContext	oldcontext;
-	List		*childs = find_inheritance_children(indexRelationId, ShareLock);
-	List		*partitions = NIL;
-
 	PreventInTransactionBlock(true, "REINDEX INDEX");
 
-	foreach (lc, childs)
+	foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
 	{
 		Oid			partoid = lfirst_oid(lc);
 
-		/* XXX: need to retrofit progress reporting into it */
-		// pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
-									 // npart++);
-
-		if (get_index_isvalid(partoid) ||
-				!RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
-			continue;
+		if (!get_index_isvalid(partoid) &&
+				RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+			ReindexRelationConcurrently(partoid, &params);
 
-		/* Save partition OID */
-		oldcontext = MemoryContextSwitchTo(ind_context);
-		partitions = lappend_oid(partitions, partoid);
-		MemoryContextSwitchTo(oldcontext);
+		pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+									 npart++);
 	}
 
-	/*
-	 * Process each partition listed in a separate transaction.  Note that
-	 * this commits and then starts a new transaction immediately.
-	 * XXX: since this is done in 2*N transactions, it could just as well
-	 * call ReindexRelationConcurrently directly
-	 */
-	ReindexMultipleInternal(partitions, &params);
-
 	/*
 	 * CIC needs to mark a partitioned index as VALID, which itself
 	 * requires setting READY, which is unset for CIC (even though
@@ -1689,6 +1669,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
 	 * partitions.
 	 * See also: validatePartitionedIndex().
 	 */
+	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
 	CommandCounterIncrement();
 	index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
-- 
2.17.0


--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"



^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* Re: [PATCH] Proof of concept for GUC improvements
@ 2022-01-12 06:55  Julien Rouhaud <[email protected]>
  0 siblings, 1 reply; 73+ messages in thread

From: Julien Rouhaud @ 2022-01-12 06:55 UTC (permalink / raw)
  To: David Christensen <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Hackers <[email protected]>; Cary Huang <[email protected]>

Hi,

On Thu, Nov 4, 2021 at 5:50 AM David Christensen <[email protected]> wrote:
>
> Hi, enclosed is a v3 [...]

According to the cfbot, the patch doesn't apply anymore and needs a
rebase: http://cfbot.cputube.org/patch_36_3290.log

> 43 out of 133 hunks FAILED -- saving rejects to file src/backend/utils/misc/guc.c.rej

I'm switching the patch to Waiting on Author.






^ permalink  raw  reply  [nested|flat] 73+ messages in thread

* Re: [PATCH] Proof of concept for GUC improvements
@ 2022-01-12 18:57  David Christensen <[email protected]>
  parent: Julien Rouhaud <[email protected]>
  0 siblings, 1 reply; 73+ messages in thread

From: David Christensen @ 2022-01-12 18:57 UTC (permalink / raw)
  To: Julien Rouhaud <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Hackers <[email protected]>; Cary Huang <[email protected]>

> Hi,
> 
> According to the cfbot, the patch doesn't apply anymore and needs a
> rebase: http://cfbot.cputube.org/patch_36_3290.log

V4 rebased attached. 



Attachments:

  [application/octet-stream] special-guc-values-v4.patch (35.1K, ../../[email protected]/2-special-guc-values-v4.patch)
  download | inline diff:
From 3e1e6219c6db94e3c69a8c5cddd6c9ab4b3aa848 Mon Sep 17 00:00:00 2001
From: David Christensen <[email protected]>
Date: Thu, 19 Aug 2021 14:33:15 -0500
Subject: [PATCH] POC: use sentinel values for parsing/outputting "special" int
 GUCs

Some GUCs include specific key values like "-1", etc, which have context beyond the actual value
involved.  Add support for providing synonyms for these values on input/output which make things a
lot easier to understand.

Add a new GUC output_special_values to enable the change of behavior on output; by default we leave
this alone, as there may be tools parsing/using these values as they currently output.

As far as I know, this "magic values" really only exists for ints; we can expand things similarly
for other types if the need arises.

For now, a non-exhaustive pass has been done through the config_int options list to identify
likely/needed values.  This will probably need to be tightened up in the future.

This code supports multiple values; it will stop at the first parsed found value and use that int
value.
---
 src/backend/utils/misc/guc.c      | 376 ++++++++++++++++++++----------
 src/include/utils/guc_tables.h    |   1 +
 src/test/regress/expected/guc.out |  45 ++++
 src/test/regress/sql/guc.sql      |  16 ++
 4 files changed, 309 insertions(+), 129 deletions(-)

diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 6fc5cbc09a..3f10fef244 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -236,6 +236,9 @@ static void assign_recovery_target_lsn(const char *newval, void *extra);
 static bool check_primary_slot_name(char **newval, void **extra, GucSource source);
 static bool check_default_with_oids(bool *newval, void **extra, GucSource source);
 
+static bool parse_special_int(const struct config_enum_entry *options, const char *value, int *result);
+static bool special_int_to_value(const struct config_enum_entry *options, int value, const char **retval);
+
 /* Private functions in guc-file.l that need to be called from guc.c */
 static ConfigVariable *ProcessConfigFileInternal(GucContext context,
 												 bool applySettings, int elevel);
@@ -566,6 +569,57 @@ extern const struct config_enum_entry recovery_target_action_options[];
 extern const struct config_enum_entry sync_method_options[];
 extern const struct config_enum_entry dynamic_shared_memory_options[];
 
+/* Some static structs for use in options which have -1 as special values;
+ * example: "disabled" or "inherited". While these are not enums per se, we
+ * are reusing the struct, with the bool field indicating whether to print the
+ * translated values on output */
+
+static const struct config_enum_entry special_auto[] = {
+	{"auto", -1, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_disabled[] = {
+	{"disabled", -1, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_disabled0[] = {
+	{"disabled", 0, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_disabled_all[] = {
+	{"disabled", -1, false},
+	{"all", 0, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_default0[] = {
+	{"default", 0, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_immediate0[] = {
+	{"immediate", 0, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_none0[] = {
+	{"none", 0, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_unlimited[] = {
+	{"unlimited", -1, false},
+	{NULL, 0, false}
+};
+
+static const struct config_enum_entry special_unlimited0[] = {
+	{"unlimited", 0, false},
+	{NULL, 0, false}
+};
+
 /*
  * GUC option variables that are exported from this module
  */
@@ -593,6 +647,8 @@ bool		check_function_bodies = true;
 bool		default_with_oids = false;
 bool		session_auth_is_superuser;
 
+bool		output_special_values = false;
+
 int			log_min_error_statement = ERROR;
 int			log_min_messages = WARNING;
 int			client_min_messages = NOTICE;
@@ -2120,6 +2176,15 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"output_special_values", PGC_USERSET, CLIENT_CONN_OTHER,
+			gettext_noop("Whether to display \"special\" values in settings display."),
+		},
+		&output_special_values,
+		false,
+		NULL, NULL, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL
@@ -2138,7 +2203,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&XLogArchiveTimeout,
 		0, 0, INT_MAX / 2,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 	{
 		{"post_auth_delay", PGC_BACKEND, DEVELOPER_OPTIONS,
@@ -2149,7 +2214,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&PostAuthDelay,
 		0, 0, INT_MAX / 1000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 	{
 		{"default_statistics_target", PGC_USERSET, QUERY_TUNING_OTHER,
@@ -2159,7 +2224,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&default_statistics_target,
 		100, 1, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"from_collapse_limit", PGC_USERSET, QUERY_TUNING_OTHER,
@@ -2172,7 +2237,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&from_collapse_limit,
 		8, 1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"join_collapse_limit", PGC_USERSET, QUERY_TUNING_OTHER,
@@ -2185,7 +2250,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&join_collapse_limit,
 		8, 1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"geqo_threshold", PGC_USERSET, QUERY_TUNING_GEQO,
@@ -2195,7 +2260,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&geqo_threshold,
 		12, 2, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"geqo_effort", PGC_USERSET, QUERY_TUNING_GEQO,
@@ -2205,7 +2270,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Geqo_effort,
 		DEFAULT_GEQO_EFFORT, MIN_GEQO_EFFORT, MAX_GEQO_EFFORT,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"geqo_pool_size", PGC_USERSET, QUERY_TUNING_GEQO,
@@ -2215,7 +2280,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Geqo_pool_size,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"geqo_generations", PGC_USERSET, QUERY_TUNING_GEQO,
@@ -2225,7 +2290,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Geqo_generations,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2237,7 +2302,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&DeadlockTimeout,
 		1000, 1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2248,7 +2313,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_standby_archive_delay,
 		30 * 1000, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited
 	},
 
 	{
@@ -2259,7 +2324,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_standby_streaming_delay,
 		30 * 1000, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited
 	},
 
 	{
@@ -2270,7 +2335,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&recovery_min_apply_delay,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2281,7 +2346,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&wal_receiver_status_interval,
 		10, 0, INT_MAX / 1000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -2292,7 +2357,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&wal_receiver_timeout,
 		60 * 1000, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -2302,7 +2367,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&MaxConnections,
 		100, 1, MAX_BACKENDS,
-		check_maxconnections, NULL, NULL
+		check_maxconnections, NULL, NULL, NULL
 	},
 
 	{
@@ -2313,7 +2378,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&ReservedBackends,
 		3, 0, MAX_BACKENDS,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2324,7 +2389,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&min_dynamic_shared_memory,
 		0, 0, (int) Min((size_t) INT_MAX, SIZE_MAX / (1024 * 1024)),
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	/*
@@ -2339,7 +2404,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&NBuffers,
 		16384, 16, INT_MAX / 2,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2350,7 +2415,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&shared_memory_size_mb,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2372,7 +2437,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&num_temp_buffers,
 		1024, 100, INT_MAX / 2,
-		check_temp_buffers, NULL, NULL
+		check_temp_buffers, NULL, NULL, NULL
 	},
 
 	{
@@ -2382,7 +2447,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&PostPortNumber,
 		DEF_PGPORT, 1, 65535,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2397,7 +2462,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Unix_socket_permissions,
 		0777, 0000, 0777,
-		NULL, NULL, show_unix_socket_permissions
+		NULL, NULL, show_unix_socket_permissions, NULL
 	},
 
 	{
@@ -2411,7 +2476,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Log_file_mode,
 		0600, 0000, 0777,
-		NULL, NULL, show_log_file_mode
+		NULL, NULL, show_log_file_mode, NULL
 	},
 
 
@@ -2426,7 +2491,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&data_directory_mode,
 		0700, 0000, 0777,
-		NULL, NULL, show_data_directory_mode
+		NULL, NULL, show_data_directory_mode, NULL
 	},
 
 	{
@@ -2439,7 +2504,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&work_mem,
 		4096, 64, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2450,7 +2515,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&maintenance_work_mem,
 		65536, 1024, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2462,7 +2527,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&logical_decoding_work_mem,
 		65536, 64, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	/*
@@ -2478,7 +2543,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_stack_depth,
 		100, 100, MAX_KILOBYTES,
-		check_max_stack_depth, assign_max_stack_depth, NULL
+		check_max_stack_depth, assign_max_stack_depth, NULL, NULL
 	},
 
 	{
@@ -2489,7 +2554,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&temp_file_limit,
 		-1, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited
 	},
 
 	{
@@ -2499,7 +2564,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&VacuumCostPageHit,
 		1, 0, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2509,7 +2574,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&VacuumCostPageMiss,
 		2, 0, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2519,7 +2584,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&VacuumCostPageDirty,
 		20, 0, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2529,7 +2594,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&VacuumCostLimit,
 		200, 1, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2539,7 +2604,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_vac_cost_limit,
 		-1, -1, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2549,7 +2614,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_files_per_process,
 		1000, 64, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	/*
@@ -2562,7 +2627,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_prepared_xacts,
 		0, 0, MAX_BACKENDS,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 #ifdef LOCK_DEBUG
@@ -2574,7 +2639,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Trace_lock_oidmin,
 		FirstNormalObjectId, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"trace_lock_table", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -2584,7 +2649,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Trace_lock_table,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 #endif
 
@@ -2596,7 +2661,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&StatementTimeout,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited0
 	},
 
 	{
@@ -2607,7 +2672,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&LockTimeout,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited0
 	},
 
 	{
@@ -2618,7 +2683,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&IdleInTransactionSessionTimeout,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited0
 	},
 
 	{
@@ -2629,7 +2694,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&IdleSessionTimeout,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited0
 	},
 
 	{
@@ -2639,7 +2704,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&vacuum_freeze_min_age,
 		50000000, 0, 1000000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2649,7 +2714,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&vacuum_freeze_table_age,
 		150000000, 0, 2000000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2659,7 +2724,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&vacuum_multixact_freeze_min_age,
 		5000000, 0, 1000000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2669,7 +2734,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&vacuum_multixact_freeze_table_age,
 		150000000, 0, 2000000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2679,7 +2744,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&vacuum_defer_cleanup_age,
 		0, 0, 1000000,			/* see ComputeXidHorizons */
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"vacuum_failsafe_age", PGC_USERSET, CLIENT_CONN_STATEMENT,
@@ -2688,7 +2753,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&vacuum_failsafe_age,
 		1600000000, 0, 2100000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"vacuum_multixact_failsafe_age", PGC_USERSET, CLIENT_CONN_STATEMENT,
@@ -2697,7 +2762,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&vacuum_multixact_failsafe_age,
 		1600000000, 0, 2100000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	/*
@@ -2712,7 +2777,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_locks_per_xact,
 		64, 10, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2724,7 +2789,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_predicate_locks_per_xact,
 		64, 10, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2735,7 +2800,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_predicate_locks_per_relation,
 		-2, INT_MIN, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2746,7 +2811,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_predicate_locks_per_page,
 		2, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2757,7 +2822,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&AuthenticationTimeout,
 		60, 1, 600,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2770,7 +2835,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&PreAuthDelay,
 		0, 0, 60,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -2781,7 +2846,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&wal_keep_size_mb,
 		0, 0, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_none0
 	},
 
 	{
@@ -2793,7 +2858,7 @@ static struct config_int ConfigureNamesInt[] =
 		&min_wal_size_mb,
 		DEFAULT_MIN_WAL_SEGS * (DEFAULT_XLOG_SEG_SIZE / (1024 * 1024)),
 		2, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2805,7 +2870,7 @@ static struct config_int ConfigureNamesInt[] =
 		&max_wal_size_mb,
 		DEFAULT_MAX_WAL_SEGS * (DEFAULT_XLOG_SEG_SIZE / (1024 * 1024)),
 		2, MAX_KILOBYTES,
-		NULL, assign_max_wal_size, NULL
+		NULL, assign_max_wal_size, NULL, NULL
 	},
 
 	{
@@ -2816,7 +2881,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&CheckPointTimeout,
 		300, 30, 86400,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2831,7 +2896,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&CheckPointWarning,
 		30, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -2842,7 +2907,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&checkpoint_flush_after,
 		DEFAULT_CHECKPOINT_FLUSH_AFTER, 0, WRITEBACK_MAX_PENDING_FLUSHES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -2853,7 +2918,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&XLOGbuffers,
 		-1, -1, (INT_MAX / XLOG_BLCKSZ),
-		check_wal_buffers, NULL, NULL
+		check_wal_buffers, NULL, NULL, special_auto
 	},
 
 	{
@@ -2864,7 +2929,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&WalWriterDelay,
 		200, 1, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2875,7 +2940,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&WalWriterFlushAfter,
 		(1024 * 1024) / XLOG_BLCKSZ, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_immediate0
 	},
 
 	{
@@ -2886,7 +2951,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&wal_skip_threshold,
 		2048, 0, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2896,7 +2961,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_wal_senders,
 		10, 0, MAX_BACKENDS,
-		check_max_wal_senders, NULL, NULL
+		check_max_wal_senders, NULL, NULL, NULL
 	},
 
 	{
@@ -2907,7 +2972,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_replication_slots,
 		10, 0, MAX_BACKENDS /* XXX? */ ,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2920,7 +2985,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_slot_wal_keep_size_mb,
 		-1, -1, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited
 	},
 
 	{
@@ -2931,7 +2996,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&wal_sender_timeout,
 		60 * 1000, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -2943,7 +3008,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&CommitDelay,
 		0, 0, 100000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_none0
 	},
 
 	{
@@ -2954,7 +3019,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&CommitSiblings,
 		5, 0, 1000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2967,7 +3032,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&extra_float_digits,
 		1, -15, 3,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -2980,7 +3045,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&log_min_duration_sample,
 		-1, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled_all
 	},
 
 	{
@@ -2992,7 +3057,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&log_min_duration_statement,
 		-1, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled_all
 	},
 
 	{
@@ -3004,7 +3069,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Log_autovacuum_min_duration,
 		600000, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled_all
 	},
 
 	{
@@ -3016,7 +3081,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&log_parameter_max_length,
 		-1, -1, INT_MAX / 2,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited
 	},
 
 	{
@@ -3028,7 +3093,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&log_parameter_max_length_on_error,
 		0, -1, INT_MAX / 2,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_unlimited
 	},
 
 	{
@@ -3039,7 +3104,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&BgWriterDelay,
 		200, 10, 10000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3049,7 +3114,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&bgwriter_lru_maxpages,
 		100, 0, INT_MAX / 2,	/* Same upper limit as shared_buffers */
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3060,7 +3125,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&bgwriter_flush_after,
 		DEFAULT_BGWRITER_FLUSH_AFTER, 0, WRITEBACK_MAX_PENDING_FLUSHES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3078,7 +3143,7 @@ static struct config_int ConfigureNamesInt[] =
 		0,
 #endif
 		0, MAX_IO_CONCURRENCY,
-		check_effective_io_concurrency, NULL, NULL
+		check_effective_io_concurrency, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3096,7 +3161,7 @@ static struct config_int ConfigureNamesInt[] =
 		0,
 #endif
 		0, MAX_IO_CONCURRENCY,
-		check_maintenance_io_concurrency, NULL, NULL
+		check_maintenance_io_concurrency, NULL, NULL, NULL
 	},
 
 	{
@@ -3107,7 +3172,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&backend_flush_after,
 		DEFAULT_BACKEND_FLUSH_AFTER, 0, WRITEBACK_MAX_PENDING_FLUSHES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3119,7 +3184,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_worker_processes,
 		8, 0, MAX_BACKENDS,
-		check_max_worker_processes, NULL, NULL
+		check_max_worker_processes, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3131,7 +3196,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_logical_replication_workers,
 		4, 0, MAX_BACKENDS,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3143,7 +3208,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_sync_workers_per_subscription,
 		2, 0, MAX_BACKENDS,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3155,7 +3220,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Log_RotationAge,
 		HOURS_PER_DAY * MINS_PER_HOUR, 0, INT_MAX / SECS_PER_MINUTE,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3167,7 +3232,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&Log_RotationSize,
 		10 * 1024, 0, INT_MAX / 1024,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3178,7 +3243,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_function_args,
 		FUNC_MAX_ARGS, FUNC_MAX_ARGS, FUNC_MAX_ARGS,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3189,7 +3254,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_index_keys,
 		INDEX_MAX_KEYS, INDEX_MAX_KEYS, INDEX_MAX_KEYS,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3200,7 +3265,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_identifier_length,
 		NAMEDATALEN - 1, NAMEDATALEN - 1, NAMEDATALEN - 1,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3211,7 +3276,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&block_size,
 		BLCKSZ, BLCKSZ, BLCKSZ,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3222,7 +3287,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&segment_size,
 		RELSEG_SIZE, RELSEG_SIZE, RELSEG_SIZE,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3233,7 +3298,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&wal_block_size,
 		XLOG_BLCKSZ, XLOG_BLCKSZ, XLOG_BLCKSZ,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3245,7 +3310,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&wal_retrieve_retry_interval,
 		5000, 1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3258,7 +3323,7 @@ static struct config_int ConfigureNamesInt[] =
 		DEFAULT_XLOG_SEG_SIZE,
 		WalSegMinSize,
 		WalSegMaxSize,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3269,7 +3334,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_naptime,
 		60, 1, INT_MAX / 1000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"autovacuum_vacuum_threshold", PGC_SIGHUP, AUTOVACUUM,
@@ -3278,7 +3343,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_vac_thresh,
 		50, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		{"autovacuum_vacuum_insert_threshold", PGC_SIGHUP, AUTOVACUUM,
@@ -3287,7 +3352,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_vac_ins_thresh,
 		1000, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled
 	},
 	{
 		{"autovacuum_analyze_threshold", PGC_SIGHUP, AUTOVACUUM,
@@ -3296,7 +3361,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_anl_thresh,
 		50, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		/* see varsup.c for why this is PGC_POSTMASTER not PGC_SIGHUP */
@@ -3308,7 +3373,7 @@ static struct config_int ConfigureNamesInt[] =
 
 		/* see vacuum_failsafe_age if you change the upper-limit value. */
 		200000000, 100000, 2000000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		/* see multixact.c for why this is PGC_POSTMASTER not PGC_SIGHUP */
@@ -3318,7 +3383,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_multixact_freeze_max_age,
 		400000000, 10000, 2000000000,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 	{
 		/* see max_connections */
@@ -3328,7 +3393,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_max_workers,
 		3, 1, MAX_BACKENDS,
-		check_autovacuum_max_workers, NULL, NULL
+		check_autovacuum_max_workers, NULL, NULL, NULL
 	},
 
 	{
@@ -3338,7 +3403,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_parallel_maintenance_workers,
 		2, 0, 1024,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3349,7 +3414,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_parallel_workers_per_gather,
 		2, 0, MAX_PARALLEL_WORKER_LIMIT,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3360,7 +3425,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&max_parallel_workers,
 		8, 0, MAX_PARALLEL_WORKER_LIMIT,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3371,7 +3436,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&autovacuum_work_mem,
 		-1, -1, MAX_KILOBYTES,
-		check_autovacuum_work_mem, NULL, NULL
+		check_autovacuum_work_mem, NULL, NULL, NULL
 	},
 
 	{
@@ -3382,7 +3447,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&old_snapshot_threshold,
 		-1, -1, MINS_PER_HOUR * HOURS_PER_DAY * 60,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled
 	},
 
 	{
@@ -3393,7 +3458,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&tcp_keepalives_idle,
 		0, 0, INT_MAX,
-		NULL, assign_tcp_keepalives_idle, show_tcp_keepalives_idle
+		NULL, assign_tcp_keepalives_idle, show_tcp_keepalives_idle, special_default0
 	},
 
 	{
@@ -3404,7 +3469,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&tcp_keepalives_interval,
 		0, 0, INT_MAX,
-		NULL, assign_tcp_keepalives_interval, show_tcp_keepalives_interval
+		NULL, assign_tcp_keepalives_interval, show_tcp_keepalives_interval, special_default0
 	},
 
 	{
@@ -3415,7 +3480,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&ssl_renegotiation_limit,
 		0, 0, 0,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3427,7 +3492,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&tcp_keepalives_count,
 		0, 0, INT_MAX,
-		NULL, assign_tcp_keepalives_count, show_tcp_keepalives_count
+		NULL, assign_tcp_keepalives_count, show_tcp_keepalives_count, special_default0
 	},
 
 	{
@@ -3438,7 +3503,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&GinFuzzySearchLimit,
 		0, 0, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3450,7 +3515,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&effective_cache_size,
 		DEFAULT_EFFECTIVE_CACHE_SIZE, 1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3461,7 +3526,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&min_parallel_table_scan_size,
 		(8 * 1024 * 1024) / BLCKSZ, 0, INT_MAX / 3,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3472,7 +3537,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&min_parallel_index_scan_size,
 		(512 * 1024) / BLCKSZ, 0, INT_MAX / 3,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3484,7 +3549,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&server_version_num,
 		PG_VERSION_NUM, PG_VERSION_NUM, PG_VERSION_NUM,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3495,7 +3560,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&log_temp_files,
 		-1, -1, INT_MAX,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, special_disabled_all
 	},
 
 	{
@@ -3506,7 +3571,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&pgstat_track_activity_query_size,
 		1024, 100, 1048576,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3517,7 +3582,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&gin_pending_list_limit,
 		4096, 64, MAX_KILOBYTES,
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3528,7 +3593,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&tcp_user_timeout,
 		0, 0, INT_MAX,
-		NULL, assign_tcp_user_timeout, show_tcp_user_timeout
+		NULL, assign_tcp_user_timeout, show_tcp_user_timeout, special_default0
 	},
 
 	{
@@ -3562,7 +3627,7 @@ static struct config_int ConfigureNamesInt[] =
 #else							/* not DISCARD_CACHES_ENABLED */
 		0, 0, 0,
 #endif							/* not DISCARD_CACHES_ENABLED */
-		NULL, NULL, NULL
+		NULL, NULL, NULL, NULL
 	},
 
 	{
@@ -3573,7 +3638,7 @@ static struct config_int ConfigureNamesInt[] =
 		},
 		&client_connection_check_interval,
 		0, 0, INT_MAX,
-		check_client_connection_check_interval, NULL, NULL
+		check_client_connection_check_interval, NULL, NULL, special_disabled0
 	},
 
 	{
@@ -3590,7 +3655,7 @@ static struct config_int ConfigureNamesInt[] =
 
 	/* End-of-list marker */
 	{
-		{NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL
+		{NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL, NULL
 	}
 };
 
@@ -6954,6 +7019,56 @@ parse_int(const char *value, int *result, int flags, const char **hintmsg)
 	return true;
 }
 
+
+/*
+ * Lookup special values from an array of chars -> int (case-sensitive).
+ * If the value is found, sets the retval value and returns
+ * true. If it's not found, return false and leave retval alone.
+ */
+bool
+parse_special_int(const struct config_enum_entry *options, const char *value,
+						   int *retval)
+{
+	const struct config_enum_entry *entry;
+
+	for (entry = options; entry && entry->name; entry++)
+	{
+		if (pg_strcasecmp(value, entry->name) == 0)
+		{
+			*retval = entry->val;
+			return true;
+		}
+	}
+
+	/* don't touch the return value in other case */
+	return false;
+}
+
+/*
+ * Lookup special values by int value and set to static string.
+ * If the value is found, sets the retval value and returns
+ * true. If it's not found, return false.
+ */
+bool
+special_int_to_value(const struct config_enum_entry *options, int value,
+						   const char **retval)
+{
+	const struct config_enum_entry *entry;
+
+	for (entry = options; entry && entry->name; entry++)
+	{
+		if (value == entry->val)
+		{
+			*retval = entry->name;
+			return true;
+		}
+	}
+
+	/* don't touch the return value in other case */
+	return false;
+}
+
+
 /*
  * Try to parse value as a floating point number in the usual format.
  * Optionally, the value can be followed by a unit name if "flags" indicates
@@ -7017,7 +7132,6 @@ parse_real(const char *value, double *result, int flags, const char **hintmsg)
 	return true;
 }
 
-
 /*
  * Lookup the name for an enum option with the selected value.
  * Should only ever be called with known-valid values, so throws
@@ -7164,8 +7278,8 @@ parse_and_validate_value(struct config_generic *record,
 				struct config_int *conf = (struct config_int *) record;
 				const char *hintmsg;
 
-				if (!parse_int(value, &newval->intval,
-							   conf->gen.flags, &hintmsg))
+				if (!(conf->special && parse_special_int(conf->special, value, &newval->intval)) &&
+					!parse_int(value, &newval->intval, conf->gen.flags, &hintmsg))
 				{
 					ereport(elevel,
 							(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -10172,7 +10286,11 @@ _ShowOption(struct config_generic *record, bool use_units)
 			{
 				struct config_int *conf = (struct config_int *) record;
 
-				if (conf->show_hook)
+				/* Special values are prioritized over show hooks */
+				if (output_special_values && conf->special && special_int_to_value(conf->special, *conf->variable, &val))
+					/* if return is true we have no special action to take here but val was set already */
+					;
+				else if (conf->show_hook)
 					val = conf->show_hook();
 				else
 				{
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index f7e54a87b7..b37122b046 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -197,6 +197,7 @@ struct config_int
 	GucIntCheckHook check_hook;
 	GucIntAssignHook assign_hook;
 	GucShowHook show_hook;
+	const struct config_enum_entry *special;
 	/* variable fields, initialized at runtime: */
 	int			reset_val;
 	void	   *reset_extra;
diff --git a/src/test/regress/expected/guc.out b/src/test/regress/expected/guc.out
index 59da91ff04..af12c4d1c3 100644
--- a/src/test/regress/expected/guc.out
+++ b/src/test/regress/expected/guc.out
@@ -813,3 +813,48 @@ set default_with_oids to f;
 -- Should not allow to set it to true.
 set default_with_oids to t;
 ERROR:  tables declared WITH OIDS are not supported
+-- tests for output_special_values and special values
+set output_special_values to t;
+set log_min_duration_statement = 100;
+show log_min_duration_statement;
+ log_min_duration_statement 
+----------------------------
+ 100ms
+(1 row)
+
+set log_min_duration_statement = -1;
+show log_min_duration_statement;
+ log_min_duration_statement 
+----------------------------
+ disabled
+(1 row)
+
+set log_min_duration_statement = disabled;
+show log_min_duration_statement;
+ log_min_duration_statement 
+----------------------------
+ disabled
+(1 row)
+
+set output_special_values to f;
+set log_min_duration_statement = 100;
+show log_min_duration_statement;
+ log_min_duration_statement 
+----------------------------
+ 100ms
+(1 row)
+
+set log_min_duration_statement = -1;
+show log_min_duration_statement;
+ log_min_duration_statement 
+----------------------------
+ -1
+(1 row)
+
+set log_min_duration_statement = disabled;
+show log_min_duration_statement;
+ log_min_duration_statement 
+----------------------------
+ -1
+(1 row)
+
diff --git a/src/test/regress/sql/guc.sql b/src/test/regress/sql/guc.sql
index c39c11388d..b6a54e0290 100644
--- a/src/test/regress/sql/guc.sql
+++ b/src/test/regress/sql/guc.sql
@@ -311,3 +311,19 @@ reset check_function_bodies;
 set default_with_oids to f;
 -- Should not allow to set it to true.
 set default_with_oids to t;
+
+-- tests for output_special_values and special values
+set output_special_values to t;
+set log_min_duration_statement = 100;
+show log_min_duration_statement;
+set log_min_duration_statement = -1;
+show log_min_duration_statement;
+set log_min_duration_statement = disabled;
+show log_min_duration_statement;
+set output_special_values to f;
+set log_min_duration_statement = 100;
+show log_min_duration_statement;
+set log_min_duration_statement = -1;
+show log_min_duration_statement;
+set log_min_duration_statement = disabled;
+show log_min_duration_statement;
-- 
2.32.0 (Apple Git-132)



^ permalink  raw  reply  [nested|flat] 73+ messages in thread