public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v9 3/8] f! progress reporting..
72+ messages / 3 participants
[nested] [flat]
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v10 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 56018745c8..28493c7931 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6293,6 +6293,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5f2541d316..0256809f06 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1142,7 +1142,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 662aff04b4..1d8a7ca774 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index c6b139d57d..d85781ae0b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 9b59a7b4a5..a8bcc599ac 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--mvpLiMfbWzRoNl4x
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v10-0004-Propagate-changes-to-indisclustered-to-child-par.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v8 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 64 +++++++++++++++++-----------
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
4 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fa58afd9d7..9a3cb5f935 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1069,7 +1069,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 9b6673867c..cb4fc350c6 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,28 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ OidIsValid(indexOid) ? PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
+
/* Do the job. */
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +218,20 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report vacuum full or cluster and number of partitions */
+ progress_val[0] = OidIsValid(indexOid) ?
+ PROGRESS_CLUSTER_COMMAND_CLUSTER :
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL;
+ progress_val[1] = list_length(rvs);
+// This is currently showing the total number of *tables*, including the
+// parent, and intermediate partitions, so the column should be renamed, or
+// we should count the number of leaf partitions
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +270,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +307,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +317,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +333,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +347,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +358,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +368,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +426,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +433,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +448,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1366,7 +1372,7 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
ReindexParams reindex_params = {0};
int i;
- /* Report that we are now swapping relation files */
+ /* Report that we are now swapping relation files XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES);
@@ -1417,13 +1423,13 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap,
else if (newrelpersistence == RELPERSISTENCE_PERMANENT)
reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT;
- /* Report that we are now reindexing relations */
+ /* Report that we are now reindexing relations XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_REBUILD_INDEX);
reindex_relation(OIDOldHeap, reindex_flags, &reindex_params);
- /* Report that we are now doing clean up */
+ /* Report that we are now doing clean up XXX */
pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE,
PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP);
@@ -1633,9 +1639,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1658,6 +1665,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ childnum++);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b632d9f2ea..d1fea532bc 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1941,7 +1941,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v9 3/8] f! progress reporting..
@ 2021-02-09 02:45 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-09 02:45 UTC (permalink / raw)
This follows the precedent of pg_stat_progress_create_index, which
simultaneously shows the progress of 1) the leaf partition, and 2) the overall
progress as partitions_done / partitions_total.
This also includes current_child_index_relid, but the create_index view
doesn't, so maybe this shouldn't, either.
---
doc/src/sgml/monitoring.sgml | 21 +++++++++++
src/backend/catalog/system_views.sql | 4 +-
src/backend/commands/cluster.c | 55 ++++++++++++++++------------
src/backend/commands/vacuum.c | 5 +++
src/include/commands/progress.h | 3 ++
src/test/regress/expected/rules.out | 4 +-
6 files changed, 67 insertions(+), 25 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 3513e127b7..dca47f6a48 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -6231,6 +6231,27 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
is <literal>rebuilding index</literal>.
</para></entry>
</row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_partitions</structfield> <type>bigint</type>
+ </para>
+ When clustering a partitioned table, this column is set to the total
+ number of partitions to be clustered.
+ <para>
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>done_partitions</structfield> <type>bigint</type>
+ </para>
+ <para>
+ When clustering a partitioned table, this column is set to the number
+ of partitions which have been clustered.
+ </para></entry>
+ </row>
+
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fc94a73a54..21c27f0437 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1070,7 +1070,9 @@ CREATE VIEW pg_stat_progress_cluster AS
S.param5 AS heap_tuples_written,
S.param6 AS heap_blks_total,
S.param7 AS heap_blks_scanned,
- S.param8 AS index_rebuild_count
+ S.param8 AS index_rebuild_count,
+ S.param18 AS partitions_total,
+ S.param19 AS partitions_done
FROM pg_stat_get_progress_info('CLUSTER') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index 0c08ac56dc..7d8d1d8a69 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -77,7 +77,7 @@ static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
-static void cluster_multiple_rels(List *rvs, int options);
+static void cluster_multiple_rels(List *rvs, int options, bool ispartitioned);
/*---------------------------------------------------------------------------
@@ -188,16 +188,26 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
/* close relation, keep lock till commit */
table_close(rel, ShareUpdateExclusiveLock);
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
{
/* Do the job. */
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
cluster_rel(tableOid, indexOid, ¶ms);
+ pgstat_progress_end_command();
}
else
{
List *rvs;
MemoryContext cluster_context;
+ int64 progress_val[2];
+ const int progress_index[2] = {
+ PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_PARTITIONS_TOTAL,
+ };
+
/* Refuse to hold strong locks in a user transaction */
PreventInTransactionBlock(isTopLevel, "CLUSTER");
@@ -206,7 +216,15 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
ALLOCSET_DEFAULT_SIZES);
rvs = get_tables_to_cluster_partitioned(cluster_context, indexOid);
- cluster_multiple_rels(rvs, params.options);
+
+ /* Report command and number of partitions */
+ progress_val[0] = PROGRESS_CLUSTER_COMMAND_CLUSTER;
+ /* XXX: rvs includes the parent, which is not a "partition" */
+ progress_val[1] = list_length(rvs);
+
+ pgstat_progress_update_multi_param(2, progress_index, progress_val);
+ cluster_multiple_rels(rvs, params.options, true);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -245,7 +263,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel)
* cluster_context.
*/
rvs = get_tables_to_cluster(cluster_context);
- cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED);
+
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_CLUSTER);
+ cluster_multiple_rels(rvs, params.options | CLUOPT_RECHECK_ISCLUSTERED, false);
+ pgstat_progress_end_command();
/* Start a new transaction for the cleanup work. */
StartTransactionCommand();
@@ -282,14 +304,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* Check for user-requested abort. */
CHECK_FOR_INTERRUPTS();
- pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid);
- if (OidIsValid(indexOid))
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_CLUSTER);
- else
- pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
- PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
-
/*
* We grab exclusive access to the target rel and index for the duration
* of the transaction. (This is redundant for the single-transaction
@@ -300,10 +314,7 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
/* If the table has gone away, we can skip processing it */
if (!OldHeap)
- {
- pgstat_progress_end_command();
return;
- }
/*
* Since we may open a new transaction for each relation, we have to check
@@ -319,7 +330,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!pg_class_ownercheck(tableOid, GetUserId()))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -334,7 +344,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (RELATION_IS_OTHER_TEMP(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -346,7 +355,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -357,7 +365,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!get_index_isclustered(indexOid))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
}
@@ -416,7 +423,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
!RelationIsPopulated(OldHeap))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -424,7 +430,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
if (!RELKIND_HAS_STORAGE(get_rel_relkind(tableOid)))
{
relation_close(OldHeap, AccessExclusiveLock);
- pgstat_progress_end_command();
return;
}
@@ -440,8 +445,6 @@ cluster_rel(Oid tableOid, Oid indexOid, ClusterParams *params)
rebuild_relation(OldHeap, indexOid, verbose);
/* NB: rebuild_relation does table_close() on OldHeap */
-
- pgstat_progress_end_command();
}
/*
@@ -1632,9 +1635,10 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid)
/* Cluster each relation in a separate transaction */
static void
-cluster_multiple_rels(List *rvs, int options)
+cluster_multiple_rels(List *rvs, int options, bool ispartitioned)
{
ListCell *lc;
+ off_t childnum = 0;
/* Commit to get out of starting transaction */
PopActiveSnapshot();
@@ -1657,6 +1661,11 @@ cluster_multiple_rels(List *rvs, int options)
cluster_rel(rvtc->tableOid, rvtc->indexOid,
&cluster_params);
+ if (ispartitioned)
+ pgstat_progress_update_param(PROGRESS_CLUSTER_PARTITIONS_DONE,
+ ++childnum);
+
+
PopActiveSnapshot();
CommitTransactionCommand();
}
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..05d52858b1 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -37,6 +37,7 @@
#include "catalog/pg_namespace.h"
#include "commands/cluster.h"
#include "commands/defrem.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -1937,7 +1938,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
cluster_params.options |= CLUOPT_VERBOSE;
/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
+ pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, relid);
+ pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND,
+ PROGRESS_CLUSTER_COMMAND_VACUUM_FULL);
cluster_rel(relid, InvalidOid, &cluster_params);
+ pgstat_progress_end_command();
}
else
table_relation_vacuum(onerel, params, vac_strategy);
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index 95ec5d02e9..13499a3e72 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -60,6 +60,9 @@
#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5
#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6
#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7
+/* Need to avoid the CREATE INDEX params */
+#define PROGRESS_CLUSTER_PARTITIONS_TOTAL 17
+#define PROGRESS_CLUSTER_PARTITIONS_DONE 18
/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */
#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b1c9b7bdfe..c888415235 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1943,7 +1943,9 @@ pg_stat_progress_cluster| SELECT s.pid,
s.param5 AS heap_tuples_written,
s.param6 AS heap_blks_total,
s.param7 AS heap_blks_scanned,
- s.param8 AS index_rebuild_count
+ s.param8 AS index_rebuild_count,
+ s.param18 AS partitions_total,
+ s.param19 AS partitions_done
FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20)
LEFT JOIN pg_database d ON ((s.datid = d.oid)));
pg_stat_progress_copy| SELECT s.pid,
--
2.17.0
--fmvA4kSBHQVZhkR6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Propagate-changes-to-indisclustered-to-child-pare.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH 2/5] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 9ab1a66971..8f4eab22eb 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1647,40 +1647,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1689,6 +1669,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--dTy3Mrz/UPE2dbVg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* [PATCH v13 02/18] f! progress reporting
@ 2021-02-15 00:31 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Justin Pryzby @ 2021-02-15 00:31 UTC (permalink / raw)
---
src/backend/commands/indexcmds.c | 33 +++++++-------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 4ac1dacd7d..d6567ec231 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -1638,40 +1638,20 @@ reindex_invalid_child_indexes(Oid indexRelationId)
.options = REINDEXOPT_CONCURRENTLY
};
- MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
- ALLOCSET_DEFAULT_SIZES);
- MemoryContext oldcontext;
- List *childs = find_inheritance_children(indexRelationId, ShareLock);
- List *partitions = NIL;
-
PreventInTransactionBlock(true, "REINDEX INDEX");
- foreach (lc, childs)
+ foreach (lc, find_inheritance_children(indexRelationId, ShareLock))
{
Oid partoid = lfirst_oid(lc);
- /* XXX: need to retrofit progress reporting into it */
- // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- // npart++);
-
- if (get_index_isvalid(partoid) ||
- !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
- continue;
+ if (!get_index_isvalid(partoid) &&
+ RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ ReindexRelationConcurrently(partoid, ¶ms);
- /* Save partition OID */
- oldcontext = MemoryContextSwitchTo(ind_context);
- partitions = lappend_oid(partitions, partoid);
- MemoryContextSwitchTo(oldcontext);
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ npart++);
}
- /*
- * Process each partition listed in a separate transaction. Note that
- * this commits and then starts a new transaction immediately.
- * XXX: since this is done in 2*N transactions, it could just as well
- * call ReindexRelationConcurrently directly
- */
- ReindexMultipleInternal(partitions, ¶ms);
-
/*
* CIC needs to mark a partitioned index as VALID, which itself
* requires setting READY, which is unset for CIC (even though
@@ -1680,6 +1660,7 @@ reindex_invalid_child_indexes(Oid indexRelationId)
* partitions.
* See also: validatePartitionedIndex().
*/
+ CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
CommandCounterIncrement();
index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0003-WIP-Add-SKIPVALID-flag-for-more-integration.patch"
^ permalink raw reply [nested|flat] 72+ messages in thread
* 2023-11-09 release announcement draft
@ 2023-11-06 22:04 Jonathan S. Katz <[email protected]>
2023-11-06 23:07 ` Re: 2023-11-09 release announcement draft Jesper Pedersen <[email protected]>
0 siblings, 1 reply; 72+ messages in thread
From: Jonathan S. Katz @ 2023-11-06 22:04 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>
Hi,
Attached is the release announcement draft for the 2023-11-09 release
(16.1 et al.).
Please review for accuracy and notable omissions. Please have all
feedback in by 2023-11-09 08:00 UTC at the latest (albeit the sooner the
better).
Thanks,
Jonathan
The PostgreSQL Global Development Group has released an update to all supported
versions of PostgreSQL, including 16.1, 15.5, 14.10, 13.13, 12.17, and 11.22
This release fixes over 55 bugs reported over the last several months.
This release includes fixes for indexes where in certain cases, we advise
reindexing. Please see the "Update" section for more details.
This is the **final release of PostgreSQL 11**. PostgreSQL 10 will no longer
receive
[security and bug fixes](https://www.postgresql.org/support/versioning/).
If you are running PostgreSQL 10 in a production environment, we suggest that
you make plans to upgrade.
For the full list of changes, please review the
[release notes](https://www.postgresql.org/docs/release/).
PostgreSQL 11 EOL Notice
------------------------
**This is the final release of PostgreSQL 11**. PostgreSQL 11 is now end-of-life
and will no longer receive security and bug fixes. If you are
running PostgreSQL 11 in a production environment, we suggest that you make
plans to upgrade to a newer, supported version of PostgreSQL. Please see our
[versioning policy](https://www.postgresql.org/support/versioning/) for more
information.
Bug Fixes and Improvements
--------------------------
This update fixes over 55 bugs that were reported in the last several months.
The issues listed below affect PostgreSQL 16. Some of these issues may also
affect other supported versions of PostgreSQL.
* Fix issue where GiST indexes had an incorrect behavior during a "page split"
operation that could lead to incorrect results in subsequent index searches.
Please [reindex](https://www.postgresql.org/docs/current/sql-reindex.html) GiST
indexes after installing this update.
* Fix issue where B-tree indexes would incorrectly de-duplicate `interval`
columns. Please [reindex](https://www.postgresql.org/docs/current/sql-reindex.html)
any B-tree index that includes an `interval` column after installing this
update.
* Provide more efficient indexing of `date`, `timestamptz`, and `timestamp`
values in BRIN indexes. While not required, we recommend
[reindexing](https://www.postgresql.org/docs/current/sql-reindex.html) BRIN
indexes that include these data types after installing this update.
* Fix for bulk table insertion into partitioned tables.
* Fix for hash-partitioned tables with multiple partition keys during step
generation and runtime pruning that could lead to crashes in some cases.
* Throw the correct error if [`pgrowlocks()`](https://www.postgresql.org/docs/current/pgrowlocks.html) is applied to a partitioned table
* Fix inconsistent rechecking of concurrently-updated rows during
[`MERGE`](https://www.postgresql.org/docs/current/sql-merge.html) when using
[`READ COMMITTED`](https://www.postgresql.org/docs/current/transaction-iso.html#XACT-READ-COMMITTED)
mode.
* Correctly identify the target table in an inherited `UPDATE`/`DELETE`/`MERGE`
even when the parent table is excluded by constraints.
* Fix over-allocation of a constructed [`tsvector`](https://www.postgresql.org/docs/current/datatype-textsearch.html#DATATYPE-TSVECTOR).
* Fix [`ALTER SUBSCRIPTION`](https://www.postgresql.org/docs/current/sql-altersubscription.html)
to apply changes in the `run_as_owner` option.
* Several fixes for [`COPY FROM`](https://www.postgresql.org/docs/current/sql-copy.html),
* Several fixes for handling torn reads with [`pg_control`](https://www.postgresql.org/docs/current/wal-internals.html).
* Fix "could not find pathkey item to sort" errors occurring while planning
aggregate functions with `ORDER BY` or `DISTINCT` options.
* When [`track_io_timing`](https://www.postgresql.org/docs/current/runtime-config-statistics.html#GUC-TRACK-IO-TIMING)
is enabled, include the time taken by relation extension operations as write
time.
* Track the dependencies of cached [`CALL`](https://www.postgresql.org/docs/current/sql-call.html)
statements, and re-plan them when needed.
* Treat out-of-memory failures as `FATAL` while reading WAL.
* Fix [`pg_dump`](https://www.postgresql.org/docs/current/app-pgdump.html) to
dump the new `run_as_owner` option of subscriptions.
* Fix [`pg_restore`](https://www.postgresql.org/docs/current/app-pgrestore.html)
so that selective restores will include both table-level and column-level ACLs
for selected tables.
* Add logic to [`pg_upgrade`](https://www.postgresql.org/docs/current/pgupgrade.html)
to check for use of obsolete data types `abstime`, `reltime`, and `tinterval`.
* Fix [`vacuumdb`](https://www.postgresql.org/docs/current/app-vacuumdb.html)
to have multiple `-N` switches actually exclude tables in multiple schemas.
* [`amcheck`](https://www.postgresql.org/docs/current/amcheck.html)
will no longer report interrupted page deletion as corruption.
* Fix `btree_gin` indexes on `interval` columns to properly return data when
using the `<` and `<=` operators.
Updating
--------
All PostgreSQL update releases are cumulative. As with other minor releases,
users are not required to dump and reload their database or use `pg_upgrade` in
order to apply this update release; you may simply shutdown PostgreSQL and
update its binaries.
We recommend [reindexing](https://www.postgresql.org/docs/current/sql-reindex.html)
certain types of indexes after you apply this update, including:
* GiST indexes
* B-tree indexes that use the `interval` data type
* BRIN indexes that use the `date`, `timestamptz`, and `timestamp`
data types
On PostgreSQL 12 and above, you can use
[`REINDEX CONCURRENTLY`](https://www.postgresql.org/docs/current/sql-reindex.html)
to avoid blocking writes to the affected index and table, for example:
```
REINDEX INDEX CONCURRENTLY your_index_name;
```
Users who have skipped one or more update releases may need to run additional
post-update steps; please see the release notes from earlier versions for
details.
For more details, please see the
[release notes](https://www.postgresql.org/docs/release/).
Links
-----
* [Download](https://www.postgresql.org/download/)
* [Release Notes](https://www.postgresql.org/docs/release/)
* [Security](https://www.postgresql.org/support/security/)
* [Versioning Policy](https://www.postgresql.org/support/versioning/)
* [PostgreSQL 16 Release Announcement](https://www.postgresql.org/about/press/)
* [Follow @postgresql on Twitter](https://twitter.com/postgresql)
* [Donate](https://www.postgresql.org/about/donate/)
If you have corrections or suggestions for this release announcement, please
send them to the [email protected]_ public
[mailing list](https://www.postgresql.org/list/).
Attachments:
[text/plain] 20231109updaterelease.md (6.4K, ../../[email protected]/2-20231109updaterelease.md)
download | inline:
The PostgreSQL Global Development Group has released an update to all supported
versions of PostgreSQL, including 16.1, 15.5, 14.10, 13.13, 12.17, and 11.22
This release fixes over 55 bugs reported over the last several months.
This release includes fixes for indexes where in certain cases, we advise
reindexing. Please see the "Update" section for more details.
This is the **final release of PostgreSQL 11**. PostgreSQL 10 will no longer
receive
[security and bug fixes](https://www.postgresql.org/support/versioning/).
If you are running PostgreSQL 10 in a production environment, we suggest that
you make plans to upgrade.
For the full list of changes, please review the
[release notes](https://www.postgresql.org/docs/release/).
PostgreSQL 11 EOL Notice
------------------------
**This is the final release of PostgreSQL 11**. PostgreSQL 11 is now end-of-life
and will no longer receive security and bug fixes. If you are
running PostgreSQL 11 in a production environment, we suggest that you make
plans to upgrade to a newer, supported version of PostgreSQL. Please see our
[versioning policy](https://www.postgresql.org/support/versioning/) for more
information.
Bug Fixes and Improvements
--------------------------
This update fixes over 55 bugs that were reported in the last several months.
The issues listed below affect PostgreSQL 16. Some of these issues may also
affect other supported versions of PostgreSQL.
* Fix issue where GiST indexes had an incorrect behavior during a "page split"
operation that could lead to incorrect results in subsequent index searches.
Please [reindex](https://www.postgresql.org/docs/current/sql-reindex.html) GiST
indexes after installing this update.
* Fix issue where B-tree indexes would incorrectly de-duplicate `interval`
columns. Please [reindex](https://www.postgresql.org/docs/current/sql-reindex.html)
any B-tree index that includes an `interval` column after installing this
update.
* Provide more efficient indexing of `date`, `timestamptz`, and `timestamp`
values in BRIN indexes. While not required, we recommend
[reindexing](https://www.postgresql.org/docs/current/sql-reindex.html) BRIN
indexes that include these data types after installing this update.
* Fix for bulk table insertion into partitioned tables.
* Fix for hash-partitioned tables with multiple partition keys during step
generation and runtime pruning that could lead to crashes in some cases.
* Throw the correct error if [`pgrowlocks()`](https://www.postgresql.org/docs/current/pgrowlocks.html) is applied to a partitioned table
* Fix inconsistent rechecking of concurrently-updated rows during
[`MERGE`](https://www.postgresql.org/docs/current/sql-merge.html) when using
[`READ COMMITTED`](https://www.postgresql.org/docs/current/transaction-iso.html#XACT-READ-COMMITTED)
mode.
* Correctly identify the target table in an inherited `UPDATE`/`DELETE`/`MERGE`
even when the parent table is excluded by constraints.
* Fix over-allocation of a constructed [`tsvector`](https://www.postgresql.org/docs/current/datatype-textsearch.html#DATATYPE-TSVECTOR).
* Fix [`ALTER SUBSCRIPTION`](https://www.postgresql.org/docs/current/sql-altersubscription.html)
to apply changes in the `run_as_owner` option.
* Several fixes for [`COPY FROM`](https://www.postgresql.org/docs/current/sql-copy.html),
* Several fixes for handling torn reads with [`pg_control`](https://www.postgresql.org/docs/current/wal-internals.html).
* Fix "could not find pathkey item to sort" errors occurring while planning
aggregate functions with `ORDER BY` or `DISTINCT` options.
* When [`track_io_timing`](https://www.postgresql.org/docs/current/runtime-config-statistics.html#GUC-TRACK-IO-TIMING)
is enabled, include the time taken by relation extension operations as write
time.
* Track the dependencies of cached [`CALL`](https://www.postgresql.org/docs/current/sql-call.html)
statements, and re-plan them when needed.
* Treat out-of-memory failures as `FATAL` while reading WAL.
* Fix [`pg_dump`](https://www.postgresql.org/docs/current/app-pgdump.html) to
dump the new `run_as_owner` option of subscriptions.
* Fix [`pg_restore`](https://www.postgresql.org/docs/current/app-pgrestore.html)
so that selective restores will include both table-level and column-level ACLs
for selected tables.
* Add logic to [`pg_upgrade`](https://www.postgresql.org/docs/current/pgupgrade.html)
to check for use of obsolete data types `abstime`, `reltime`, and `tinterval`.
* Fix [`vacuumdb`](https://www.postgresql.org/docs/current/app-vacuumdb.html)
to have multiple `-N` switches actually exclude tables in multiple schemas.
* [`amcheck`](https://www.postgresql.org/docs/current/amcheck.html)
will no longer report interrupted page deletion as corruption.
* Fix `btree_gin` indexes on `interval` columns to properly return data when
using the `<` and `<=` operators.
Updating
--------
All PostgreSQL update releases are cumulative. As with other minor releases,
users are not required to dump and reload their database or use `pg_upgrade` in
order to apply this update release; you may simply shutdown PostgreSQL and
update its binaries.
We recommend [reindexing](https://www.postgresql.org/docs/current/sql-reindex.html)
certain types of indexes after you apply this update, including:
* GiST indexes
* B-tree indexes that use the `interval` data type
* BRIN indexes that use the `date`, `timestamptz`, and `timestamp`
data types
On PostgreSQL 12 and above, you can use
[`REINDEX CONCURRENTLY`](https://www.postgresql.org/docs/current/sql-reindex.html)
to avoid blocking writes to the affected index and table, for example:
```
REINDEX INDEX CONCURRENTLY your_index_name;
```
Users who have skipped one or more update releases may need to run additional
post-update steps; please see the release notes from earlier versions for
details.
For more details, please see the
[release notes](https://www.postgresql.org/docs/release/).
Links
-----
* [Download](https://www.postgresql.org/download/)
* [Release Notes](https://www.postgresql.org/docs/release/)
* [Security](https://www.postgresql.org/support/security/)
* [Versioning Policy](https://www.postgresql.org/support/versioning/)
* [PostgreSQL 16 Release Announcement](https://www.postgresql.org/about/press/)
* [Follow @postgresql on Twitter](https://twitter.com/postgresql)
* [Donate](https://www.postgresql.org/about/donate/)
If you have corrections or suggestions for this release announcement, please
send them to the [email protected]_ public
[mailing list](https://www.postgresql.org/list/).
[application/pgp-signature] OpenPGP_signature.asc (840B, ../../[email protected]/3-OpenPGP_signature.asc)
download
^ permalink raw reply [nested|flat] 72+ messages in thread
* Re: 2023-11-09 release announcement draft
2023-11-06 22:04 2023-11-09 release announcement draft Jonathan S. Katz <[email protected]>
@ 2023-11-06 23:07 ` Jesper Pedersen <[email protected]>
0 siblings, 0 replies; 72+ messages in thread
From: Jesper Pedersen @ 2023-11-06 23:07 UTC (permalink / raw)
To: Jonathan S. Katz <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
Hi,
On 11/6/23 17:04, Jonathan S. Katz wrote:
> Attached is the release announcement draft for the 2023-11-09 release
> (16.1 et al.).
>
> Please review for accuracy and notable omissions. Please have all
> feedback in by 2023-11-09 08:00 UTC at the latest (albeit the sooner
> the better).
s/PostgreSQL 10/PostgreSQL 11/g
Best regards,
Jesper
^ permalink raw reply [nested|flat] 72+ messages in thread
end of thread, other threads:[~2023-11-06 23:07 UTC | newest]
Thread overview: 72+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v10 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v8 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-09 02:45 [PATCH v9 3/8] f! progress reporting.. Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH 2/5] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2021-02-15 00:31 [PATCH v13 02/18] f! progress reporting Justin Pryzby <[email protected]>
2023-11-06 22:04 2023-11-09 release announcement draft Jonathan S. Katz <[email protected]>
2023-11-06 23:07 ` Re: 2023-11-09 release announcement draft Jesper Pedersen <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox